steep 1.8.0.dev.2 → 1.8.0.pre.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eff62e36ae34f76bafc5eb5ec8425ba373dd9ab64ae880537d65b96b4581970c
-  data.tar.gz: b0dc7b3d10b83a0e188a211f40c3a91eb3e146138da328079c84639b92c87bb8
+  metadata.gz: c87015f545300c9c126b0121aa9534db083c157fbd791b7117006b6683bdc05d
+  data.tar.gz: 6bb606ac1b4cc9cdd6364745c02f878714cdf2ca2d61079376a3ea50fbf4b9ba
 SHA512:
-  metadata.gz: 6e5d7e9e19bf782a8a03842eed077030a9f72eeb5f570ef23110658916f2d4c93c1090e3f3f6017957c80ef1a74c4c7a33bbc295316fe4afb47309cf47c2bcac
-  data.tar.gz: 66a14f3ec60c294a90e85bde099eb40224f22a99493f121bd8378e288affd7ad32a7ee8e1aff23915dd373768c4a35657fd92e14c7afe9e9cbe244c18d36d96c
+  metadata.gz: 4787c76c6049f2e1a2a9856d26dd0157419a2f4258a185559d0fe676500380089d75b30f916e353836f0e6af4eab16e2d977a9be5798bcfae1d13f8902d21491
+  data.tar.gz: 462bb7747747fd5a34bf3205cd71d94f22beb4efd4407e5e59fbc31c049583b8350cdac262591319ad2ed09d802d94e3ff42d2f9c54ba0addff5547c7258f898

data/CHANGELOG.md

@@ -1,5 +1,38 @@
 # CHANGELOG
 
+## 1.8.0.pre.1 (2024-09-17)
+
+### Type checker core
+
+* Fix some subtyping problems ([#1221](https://github.com/soutaro/steep/pull/1221))
+* Support optional keys in record types ([#1223](https://github.com/soutaro/steep/pull/1223))
+* Revert implicit generic upper bound ([#1220](https://github.com/soutaro/steep/pull/1220))
+* Improve generics ([#1216](https://github.com/soutaro/steep/pull/1216))
+* Delete meta data from `MethodType` and `Types::*` ([#1201](https://github.com/soutaro/steep/pull/1201))
+* Delete `ContextArray` ([#1199](https://github.com/soutaro/steep/pull/1199))
+* Fix shape calculation error ([#1197](https://github.com/soutaro/steep/pull/1197))
+* Skip `#:` syntax in `Data.define` and `Struct.new` ([#1196](https://github.com/soutaro/steep/pull/1196))
+* Support literals for Rational and Complex ([#1178](https://github.com/soutaro/steep/pull/1178))
+* Ignore type assertions on arguments/receiver on `def`/`defs` ([#1179](https://github.com/soutaro/steep/pull/1179))
+* Ignore `#:` annotation on `attr_*` calls ([#1176](https://github.com/soutaro/steep/pull/1176))
+
+### Commandline tool
+
+* Improve performance when there are many files ([#1184](https://github.com/soutaro/steep/pull/1184))
+* Fix type errors ([#1183](https://github.com/soutaro/steep/pull/1183))
+
+### Language server
+
+* Make type checking a LSP request, not a LSP notification ([#1218](https://github.com/soutaro/steep/pull/1218))
+* No `timeout: nil` ([#1217](https://github.com/soutaro/steep/pull/1217))
+* Print `loading project` message via LSP ([#1213](https://github.com/soutaro/steep/pull/1213))
+* Load files in main process ([#1206](https://github.com/soutaro/steep/pull/1206))
+
+### Miscellaneous
+
+* Drop Ruby 3.0 ([#1225](https://github.com/soutaro/steep/pull/1225))
+* Add `bin/steep-check` for profiling ([#1198](https://github.com/soutaro/steep/pull/1198))
+
 ## 1.7.1 (2024-06-12)
 
 ### Type checker core

data/bin/mem_graph.rb

@@ -0,0 +1,67 @@
+class MemGraph
+  attr_reader :edges
+
+  attr_reader :checked
+
+  attr_reader :generation
+
+  def initialize(generation)
+    @generation = generation
+    @edges = []
+    @checked = Set.new.compare_by_identity
+    @checked << self
+    @checked << edges
+    @checked << checked
+  end
+
+  IVARS = Object.instance_method(:instance_variables)
+  IVGET = Object.instance_method(:instance_variable_get)
+
+  def traverse(object)
+    return if checked.include?(object)
+    checked << object
+
+    case object
+    when Array
+      object.each do |value|
+        insert_edge(object, value)
+        traverse(value)
+      end
+    when Hash
+      object.each do |key, value|
+        insert_edge(object, key)
+        insert_edge(object, value)
+
+        traverse(key)
+        traverse(value)
+      end
+    else
+      IVARS.bind_call(object).each do |name|
+        if name.is_a?(Symbol)
+          value = IVGET.bind_call(object, name)
+          traverse(value)
+          insert_edge(object, value)
+        else
+          STDERR.puts "Unexpected instance variable name: #{name} in #{object.class}"
+        end
+      end
+    end
+  end
+
+  def insert_edge(source, dest)
+    case dest
+    when Integer, Symbol, nil, true, false, Float
+    else
+      edges << [
+        "#{source.class}(#{source.__id__})",
+        "#{dest.class}(#{dest.__id__})",
+      ]
+    end
+  end
+
+  def dot
+    "digraph G {\n" + edges.uniq.map do |source, dest|
+      "  #{source} -> #{dest};"
+    end.join("\n") + "}"
+  end
+end

data/bin/mem_prof.rb

@@ -0,0 +1,102 @@
+require "objspace"
+
+class MemProf
+  attr_reader :generation
+
+  def initialize
+  end
+
+  def self.trace(io: STDOUT, &block)
+    profiler = MemProf.new
+    profiler.start
+
+    begin
+      ret = yield
+    rescue
+      ObjectSpace.trace_object_allocations_stop
+      ObjectSpace.trace_object_allocations_clear
+      raise
+    end
+
+    allocated, retained, collected = profiler.stop
+
+    counts = {}
+    collected.each do |id, entry|
+      counts[entry] ||= 0
+      counts[entry] += 1
+    end
+
+    counts.keys.sort_by {|entry| -counts[entry] }.take(200).each do |entry|
+      count = counts.fetch(entry)
+      io.puts "#{entry[0]},#{entry[1]},#{entry[2]},#{count}"
+    end
+
+    STDERR.puts "Total allocated: #{allocated.size}"
+    STDERR.puts "Total retained: #{retained.size}"
+    STDERR.puts "Total collected: #{collected.size}"
+
+    ret
+  end
+
+  def start
+    GC.disable
+    3.times { GC.start }
+    GC.start
+
+    @generation = GC.count
+    ObjectSpace.trace_object_allocations_start
+  end
+
+  def stop
+    ObjectSpace.trace_object_allocations_stop
+
+    allocated = objects()
+    retained = {}
+
+    GC.enable
+    GC.start
+    GC.start
+    GC.start
+
+    ObjectSpace.each_object do |obj|
+      next unless ObjectSpace.allocation_generation(obj) == generation
+      if o = allocated[obj.__id__]
+        retained[obj.__id__] = o
+      end
+    end
+
+    # ObjectSpace.trace_object_allocations_clear
+
+    collected = {}
+    allocated.each do |id, state|
+      collected[id] = state unless retained.key?(id)
+    end
+
+    [allocated, retained, collected]
+  end
+
+  def objects(hash = {})
+    ObjectSpace.each_object do |obj|
+      next unless ObjectSpace.allocation_generation(obj) == generation
+
+      file = ObjectSpace.allocation_sourcefile(obj) || "(no name)"
+      line = ObjectSpace.allocation_sourceline(obj)
+      klass = object_class(obj)
+
+      hash[obj.__id__] = [file, line, klass]
+    end
+
+    hash
+  end
+
+  KERNEL_CLASS_METHOD = Kernel.instance_method(:class)
+  def object_class(obj)
+    klass = obj.class rescue nil
+
+    unless Class === klass
+      # attempt to determine the true Class when .class returns something other than a Class
+      klass = KERNEL_CLASS_METHOD.bind_call(obj)
+    end
+    klass
+  end
+end

data/bin/stackprof_test.rb

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require "stackprof"
+
+mode = (ENV["STEEP_STACKPROF_MODE"] || :cpu).to_sym
+out = ENV["STEEP_STACKPROF_OUT"] || "tmp/stackprof-#{mode}-test.dump"
+interval = ENV["STEEP_STACKPROF_INTERVAL"]&.to_i || 1000
+
+STDERR.puts "Running profiler: mode => #{mode}, out => #{out}"
+StackProf.run(mode: mode, out: out, raw: true, interval: interval) do
+  # 10.times do
+  #   1_000.times do
+  #     Array.new(1_000_000)
+  #   end
+  #   sleep 0.1
+  # end
+
+  sleep 5
+end

data/bin/steep-check.rb

@@ -0,0 +1,251 @@
+#!/usr/bin/env ruby
+
+require 'pathname'
+
+$LOAD_PATH << Pathname(__dir__) + "../lib"
+
+require 'steep'
+require "fileutils"
+require "optparse"
+
+puts <<MESSAGE
+This is a single-threaded and one-process type checker for profiling.
+It runs really slow because it's not parallelized, but it's easier to profiling and analyzing the result.
+
+$ bundle exec bin/steep-check --target=app
+$ bundle exec bin/steep-check --target=app files...
+
+MESSAGE
+
+steep_file_path = Pathname("Steepfile")
+target_name = nil #: Symbol?
+profile_mode = :none #: Symbol?
+
+OptionParser.new do |opts|
+  opts.on("--steepfile=FILE") do |file|
+    steep_file_path = Pathname(file)
+  end
+  opts.on("--target=TARGET") do |target|
+    target_name = target.to_sym
+  end
+  opts.on("--profile=MODE", "vernier, memory, stackprof, none, majo, memory2, dumpall") do |mode|
+    profile_mode = mode.to_sym
+  end
+end.parse!(ARGV)
+
+command_line_args = ARGV.dup
+
+class Command
+  include Steep
+
+  attr_reader :project, :target
+
+  def initialize(steep_file_path:, target_name:)
+    @project = Project.new(steepfile_path: steep_file_path).tap do |project|
+      Project::DSL.parse(project, steep_file_path.read, filename: steep_file_path.to_s)
+    end
+
+    if target_name
+      @target = project.targets.find { _1.name == target_name}
+    else
+      @target = project.targets[0]
+    end
+
+    unless target
+      raise "!! Cannot find a target with #{target_name}: targets=[#{project.targets.map(&:name).join(", ")}]"
+    end
+  end
+
+  def load_signatures()
+    file_loader = Steep::Services::FileLoader.new(base_dir: project.base_dir)
+
+    signature_files = {}
+    file_loader.each_path_in_patterns(target.signature_pattern) do |path|
+      signature_files[path] = path.read
+    end
+
+    target.options.load_collection_lock
+
+    signature_service = Steep::Project::Target.construct_env_loader(options: target.options, project: project).yield_self do |loader|
+      Steep::Services::SignatureService.load_from(loader)
+    end
+
+    env = signature_service.latest_env
+
+    new_decls = Set[]
+
+    signature_files.each do |path, content|
+      buffer = RBS::Buffer.new(name: path.to_s, content: content)
+      buffer, dirs, decls = RBS::Parser.parse_signature(buffer)
+      env.add_signature(buffer: buffer, directives: dirs, decls: decls)
+      new_decls.merge(decls)
+    end
+
+    env.resolve_type_names(only: new_decls)
+  end
+
+  def type_check_files(command_line_args, env)
+    file_loader = Steep::Services::FileLoader.new(base_dir: project.base_dir)
+
+    source_files = {}
+    file_loader.each_path_in_patterns(target.source_pattern, command_line_args) do |path|
+      source_files[path] = path.read
+    end
+
+    definition_builder = RBS::DefinitionBuilder.new(env: env)
+    factory = AST::Types::Factory.new(builder: definition_builder)
+    builder = Interface::Builder.new(factory)
+    subtyping = Subtyping::Check.new(builder: builder)
+
+    typings = {}
+
+    source_files.each do |path, content|
+      source = Source.parse(content, path: path, factory: factory)
+
+      self_type = AST::Builtin::Object.instance_type
+
+      annotations = source.annotations(block: source.node, factory: factory, context: nil)
+      resolver = RBS::Resolver::ConstantResolver.new(builder: factory.definition_builder)
+      const_env = TypeInference::ConstantEnv.new(factory: factory, context: nil, resolver: resolver)
+
+      type_env = TypeInference::TypeEnvBuilder.new(
+        TypeInference::TypeEnvBuilder::Command::ImportGlobalDeclarations.new(factory),
+        TypeInference::TypeEnvBuilder::Command::ImportInstanceVariableAnnotations.new(annotations),
+        TypeInference::TypeEnvBuilder::Command::ImportConstantAnnotations.new(annotations),
+        TypeInference::TypeEnvBuilder::Command::ImportLocalVariableAnnotations.new(annotations)
+      ).build(TypeInference::TypeEnv.new(const_env))
+
+      context = TypeInference::Context.new(
+        block_context: nil,
+        method_context: nil,
+        module_context: TypeInference::Context::ModuleContext.new(
+          instance_type: AST::Builtin::Object.instance_type,
+          module_type: AST::Builtin::Object.module_type,
+          implement_name: nil,
+          nesting: nil,
+          class_name: AST::Builtin::Object.module_name,
+          instance_definition: factory.definition_builder.build_instance(AST::Builtin::Object.module_name),
+          module_definition: factory.definition_builder.build_singleton(AST::Builtin::Object.module_name)
+        ),
+        break_context: nil,
+        self_type: self_type,
+        type_env: type_env,
+        call_context: TypeInference::MethodCall::TopLevelContext.new(),
+        variable_context: TypeInference::Context::TypeVariableContext.empty
+      )
+
+      typing = Typing.new(source: source, root_context: context, cursor: nil)
+      construction = TypeConstruction.new(checker: subtyping, source: source, annotations: annotations, context: context, typing: typing)
+
+      construction.synthesize(source.node)
+      typings[path] = typing
+    end
+
+    typings
+  end
+end
+
+command = Command.new(steep_file_path: Pathname.pwd + steep_file_path, target_name: target_name)
+
+puts ">> Loading RBS files..."
+env = command.load_signatures()
+
+puts ">> Type checking files with #{profile_mode}..."
+
+typings = nil
+
+GC.start(immediate_sweep: true, immediate_mark: true, full_mark: true)
+# GC.config[:rgengc_allow_major_gc] = false
+
+case profile_mode
+when :vernier
+  require "vernier"
+  out = Pathname.pwd + "tmp/typecheck-#{Process.pid}.vernier.json"
+  puts ">> Profiling with vernier: #{out}"
+  Vernier.profile(out: out.to_s) do
+    typings = command.type_check_files(command_line_args, env)
+  end
+
+when :memory
+  require 'memory_profiler'
+  out = Pathname.pwd + "tmp/typecheck-#{Process.pid}.memory.txt"
+  puts ">> Profiling with memory_profiler: #{out}"
+  classes = nil
+  report = MemoryProfiler.report(trace: classes) do
+    typings = command.type_check_files(command_line_args, env)
+  end
+  report.pretty_print(to_file: out, detailed_report: true, scale_bytes: true, retained_strings: false, allocated_strings: false)
+
+when :memory2
+  require_relative 'mem_prof'
+  out = Pathname.pwd + "tmp/typecheck-#{Process.pid}.memory2.csv"
+  puts ">> Profiling with mem_prof: #{out}"
+  generation = nil
+  out.open("w") do |io|
+    MemProf.trace(io: io) do
+      generation = GC.count
+      typings = command.type_check_files(command_line_args, env)
+    end
+  end
+
+  require_relative 'mem_graph'
+  graph = MemGraph.new(generation)
+  ObjectSpace.each_object do |obj|
+    if ObjectSpace.allocation_generation(obj) == generation
+      graph.traverse(obj)
+    end
+  end
+  (Pathname.pwd + "objects-#{Process.pid}.dot").write(graph.dot)
+
+when :stackprof
+  require "stackprof"
+  out = Pathname.pwd + "tmp/typecheck-#{Process.pid}.stackprof"
+  puts ">> Profiling with stackprof: #{out}"
+  StackProf.run(out: out, raw: true, interval: 1000) do
+    typings = command.type_check_files(command_line_args, env)
+  end
+
+when :majo
+  require "majo"
+  out = Pathname.pwd + "tmp/typecheck-#{Process.pid}.majo.csv"
+  puts ">> Profiling with majo: #{out}"
+
+  result = Majo.run do
+    typings = command.type_check_files(command_line_args, env)
+  end
+
+  out.open("w") do  |io|
+    result.report(out: io, formatter: :csv)
+  end
+
+when :dumpall
+  require "objspace"
+  out = Pathname.pwd + "tmp/dumpall-#{Process.pid}.json"
+  puts ">> Profiling with dumpall: #{out}"
+  ObjectSpace.trace_object_allocations_start
+  typings = command.type_check_files(command_line_args, env)
+  out.open('w+') do |io|
+    ObjectSpace.dump_all(output: io)
+  end
+
+when :none
+
+  pp rbs_method_types: ObjectSpace.each_object(RBS::MethodType).count
+  GC.start(immediate_sweep: true, immediate_mark: true, full_mark: true)
+
+  pp defs: ObjectSpace.each_object(RBS::AST::Members::MethodDefinition).count
+  pp aliases: ObjectSpace.each_object(RBS::AST::Members::Alias).count
+  pp attr_reader: ObjectSpace.each_object(RBS::AST::Members::AttrReader).count
+  pp attr_writer: ObjectSpace.each_object(RBS::AST::Members::AttrWriter).count
+  pp attr_accessor: ObjectSpace.each_object(RBS::AST::Members::AttrAccessor).count
+
+  Steep.measure("type check", level: :fatal) do
+    GC.disable
+    typings = command.type_check_files(command_line_args, env)
+
+    pp steep_method_types: ObjectSpace.each_object(Steep::Interface::MethodType).count, rbs_method_types: ObjectSpace.each_object(RBS::MethodType).count
+    pp any: ObjectSpace.each_object(Steep::AST::Types::Any).count, void: ObjectSpace.each_object(Steep::AST::Types::Void).count, self: ObjectSpace.each_object(Steep::AST::Types::Self).count
+  end
+end
+
+typings.size

data/lib/steep/annotation_parser.rb

@@ -91,7 +91,7 @@ module Steep
           name = match[:name] or raise
           type = match[:type] or raise
 
-          method_type = factory.method_type(RBS::Parser.parse_method_type(type) || raise, method_decls: Set[])
+          method_type = factory.method_type(RBS::Parser.parse_method_type(type) || raise)
 
           AST::Annotation::MethodType.new(name: name.to_sym,
                                           type: method_type,

data/lib/steep/ast/builtin.rb

@@ -67,23 +67,23 @@ module Steep
       Proc = Type.new("::Proc")
 
       def self.nil_type
-        AST::Types::Nil.new
+        AST::Types::Nil.instance
       end
 
       def self.any_type
-        AST::Types::Any.new
+        AST::Types::Any.instance
       end
 
       def self.bool_type
-        AST::Types::Boolean.new
+        AST::Types::Boolean.instance
       end
 
       def self.bottom_type
-        AST::Types::Bot.new
+        AST::Types::Bot.instance
       end
 
       def self.top_type
-        AST::Types::Top.new
+        AST::Types::Top.instance
       end
 
       def self.optional(type)

data/lib/steep/ast/node/type_application.rb

@@ -27,14 +27,15 @@ module Steep
         def types(context, subtyping, type_vars)
           resolver = RBS::Resolver::TypeNameResolver.new(subtyping.factory.env)
 
-          # @type var types: Array[Types::t]
+          # @type var types: Array[LocatedValue[Types::t]]
           types = []
 
           loc = type_location
 
           while true
-            ty = RBS::Parser.parse_type(loc.buffer, range: loc.range, variables: type_vars) or break
-            ty = ty.map_type_name {|name| resolver.resolve(name, context: context) || name.absolute! }
+            rbs_ty = RBS::Parser.parse_type(loc.buffer, range: loc.range, variables: type_vars) or break
+            rbs_loc = rbs_ty.location or raise
+            ty = rbs_ty.map_type_name {|name| resolver.resolve(name, context: context) || name.absolute! }
 
             validator = Signature::Validator.new(checker: subtyping)
             validator.rescue_validation_errors do
@@ -46,11 +47,11 @@ module Steep
             end
 
             ty = subtyping.factory.type(ty)
-            types << ty
+            types << LocatedValue.new(value: ty, location: rbs_loc)
 
-            match = RBS::Location.new(loc.buffer, ty.location.end_pos, type_location.end_pos).source.match(/\A\s*,\s*/) or break
+            match = RBS::Location.new(loc.buffer, rbs_loc.end_pos, type_location.end_pos).source.match(/\A\s*,\s*/) or break
             offset = match.length
-            loc = RBS::Location.new(loc.buffer, ty.location.end_pos + offset, type_location.end_pos)
+            loc = RBS::Location.new(loc.buffer, rbs_loc.end_pos + offset, type_location.end_pos)
           end
 
           types

data/lib/steep/ast/types/any.rb

@@ -2,11 +2,7 @@ module Steep
   module AST
     module Types
       class Any
-        attr_reader :location
-
-        def initialize(location: nil)
-          @location = location
-        end
+        extend SharedInstance
 
         def ==(other)
           other.is_a?(Any)
@@ -33,10 +29,6 @@ module Steep
         def level
           [1]
         end
-
-        def with_location(new_location)
-          self.class.new(location: new_location)
-        end
       end
     end
   end

data/lib/steep/ast/types/boolean.rb

@@ -2,12 +2,8 @@ module Steep
   module AST
     module Types
       class Boolean
-        attr_reader :location
-
-        def initialize(location: nil)
-          @location = location
-        end
-
+        extend SharedInstance
+        
         def ==(other)
           other.is_a?(Boolean)
         end
@@ -34,17 +30,13 @@ module Steep
           [0]
         end
 
-        def with_location(new_location)
-          self.class.new(location: new_location)
-        end
-
         def back_type
-          Union.build(types:
-                      [
-                        Builtin::TrueClass.instance_type,
-                        Builtin::FalseClass.instance_type
-                      ],
-                      location: location)
+          Union.build(
+            types: [
+              Builtin::TrueClass.instance_type,
+              Builtin::FalseClass.instance_type
+            ]
+          )
         end
       end
     end

data/lib/steep/ast/types/bot.rb

@@ -2,12 +2,8 @@ module Steep
   module AST
     module Types
       class Bot
-        attr_reader :location
-
-        def initialize(location: nil)
-          @location = location
-        end
-
+        extend SharedInstance
+        
         def ==(other)
           other.is_a?(Bot)
         end
@@ -33,10 +29,6 @@ module Steep
         def level
           [2]
         end
-
-        def with_location(new_location)
-          self.class.new(location: new_location)
-        end
       end
     end
   end

data/lib/steep/ast/types/class.rb

@@ -2,15 +2,7 @@ module Steep
   module AST
     module Types
       class Class
-        attr_reader :location
-
-        def initialize(location: nil)
-          @location = location
-        end
-
-        def self.instance
-          @instance ||= new()
-        end
+        extend SharedInstance
 
         def to_s
           "class"
@@ -45,10 +37,6 @@ module Steep
         def level
           [0]
         end
-
-        def with_location(new_location)
-          self.class.new(location: new_location)
-        end
       end
     end
   end