sorbet 0.4.4250 → 0.4.4253

data/lib/gem_loader.rbi

@@ -0,0 +1,15 @@
+# typed: true
+
+# Write this as an RBI because the gem_loader.rb file itself cannot be typed.
+# By nature, it references constants in other gems in an attempt to load them.
+# They will not load in this project. It's marked as `typed: ignore`, and this
+# file exists so that we can call into gem_loader from typed code.
+
+class Sorbet::Private::GemLoader
+  sig {params(gem: String).void}
+  def self.require_gem(gem)
+  end
+
+  sig {void}
+  def self.require_all_gems; end
+end

data/lib/hidden-definition-finder.rb

@@ -0,0 +1,441 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+# typed: true
+
+class Sorbet; end
+module Sorbet::Private; end
+
+require_relative './t'
+require_relative './step_interface'
+require_relative './serialize'
+require_relative './constant_cache'
+require_relative './require_everything'
+require_relative './real_stdlib'
+
+require 'fileutils'
+require 'json'
+require 'set'
+require 'tmpdir'
+
+class Sorbet::Private::HiddenMethodFinder
+  PATH = "sorbet/rbi/hidden-definitions/"
+  TMP_PATH = Dir.mktmpdir + "/"
+  TMP_RBI = TMP_PATH + "reflection.rbi"
+  DIFF_RBI = TMP_PATH + "hidden.rbi.tmp"
+  RBI_CONSTANTS = TMP_PATH + "reflection.json"
+  RBI_CONSTANTS_ERR = RBI_CONSTANTS + ".err"
+  SOURCE_CONSTANTS = TMP_PATH + "from-source.json"
+  SOURCE_CONSTANTS_ERR = SOURCE_CONSTANTS + ".err"
+
+  HIDDEN_RBI = PATH + "hidden.rbi"
+  ERRORS_RBI = PATH + "errors.txt"
+
+  HEADER = Sorbet::Private::Serialize.header('autogenerated', 'hidden-definitions')
+
+  include Sorbet::Private::StepInterface
+
+  def self.main
+    self.new.main
+  end
+
+  def main
+    mk_dir
+    require_everything
+    classes, aliases = all_modules_and_aliases
+    gen_source_rbi(classes, aliases)
+    rm_rbis
+    write_constants
+    source, rbi = read_constants
+    write_diff(source, rbi)
+    split_rbi
+    rm_dir
+  end
+
+  def mk_dir
+    FileUtils.mkdir_p(PATH) unless Dir.exist?(PATH)
+  end
+
+  def rm_dir
+    FileUtils.rm_r(TMP_PATH)
+  end
+
+  def require_everything
+    puts "Requiring all of your code"
+    Sorbet::Private::RequireEverything.require_everything
+  end
+
+  def constant_cache
+    @cache ||= Sorbet::Private::ConstantLookupCache.new
+    @cache
+  end
+
+
+  def all_modules_and_aliases
+    puts "Naming all Modules"
+    [constant_cache.all_module_names.sort, constant_cache.all_module_aliases]
+  end
+
+  def real_name(mod)
+    constant_cache.name_by_class(mod)
+  end
+
+  def gen_source_rbi(classes, aliases)
+    puts "Generating #{TMP_RBI} with #{classes.count} modules and #{aliases.count} aliases"
+    serializer = Sorbet::Private::Serialize.new(constant_cache)
+    buffer = []
+    buffer << Sorbet::Private::Serialize.header
+
+    # should we do something with these errors?
+    capture_stderr do
+      classes.each do |class_name|
+        buffer << serializer.class_or_module(class_name)
+      end
+      aliases.each do |base, other_names|
+        other_names.each do |other_name|
+          buffer << serializer.alias(base, other_name)
+        end
+      end
+    end
+    File.write(TMP_RBI, buffer.join("\n"))
+  end
+
+  def write_constants
+    puts "Printing your code's symbol table into #{SOURCE_CONSTANTS}"
+    io = IO.popen(
+      [
+        File.realpath("#{__dir__}/../bin/srb"),
+        'tc',
+        '--print=symbol-table-json',
+        '--stdout-hup-hack',
+        '--silence-dev-message',
+        '--no-error-count',
+      ],
+      err: SOURCE_CONSTANTS_ERR
+    )
+    File.write(SOURCE_CONSTANTS, io.read)
+    io.close
+    raise "Your source can't be read by Sorbet.\nYou can try `srb tc --print=symbol-table-json -vvvv --max-threads 1` and hopefully the last file it is processing before it dies is the culprit.\nIf not, maybe the errors in this file will help: #{SOURCE_CONSTANTS_ERR}" if File.read(SOURCE_CONSTANTS).empty?
+
+    puts "Printing #{TMP_RBI}'s symbol table into #{RBI_CONSTANTS}"
+    # Change dir to deal with you having a sorbet/config in your cwd
+    Dir.chdir(TMP_PATH) do
+      io = IO.popen(
+        [
+          File.realpath("#{__dir__}/../bin/srb"),
+          'tc',
+          '--print=symbol-table-json',
+          # Method redefined with mismatched argument is ok since sometime
+          # people monkeypatch over method
+          '--error-black-list=4010',
+          # Redefining constant is needed because we serialize things both as
+          # aliases and in-class constants.
+          '--error-black-list=4012',
+          # Invalid nesting is ok because we don't generate all the intermediate
+          # namespaces for aliases
+          '--error-black-list=4015',
+          '--stdout-hup-hack',
+          '--silence-dev-message',
+          '--no-error-count',
+          TMP_RBI,
+        ],
+        err: RBI_CONSTANTS_ERR
+      )
+    end
+    File.write(RBI_CONSTANTS, io.read)
+    io.close
+    raise "#{TMP_RBI} had unexpected errors. Check this file for a clue: #{RBI_CONSTANTS_ERR}" unless $?.success?
+  end
+
+  def read_constants
+    puts "Reading #{SOURCE_CONSTANTS}"
+    source = JSON.parse(File.read(SOURCE_CONSTANTS))
+    puts "Reading #{RBI_CONSTANTS}"
+    rbi = JSON.parse(File.read(RBI_CONSTANTS))
+    [source, rbi]
+  end
+
+  def write_diff(source, rbi)
+    puts "Building rbi id to symbol map"
+    rbi_symbols = symbols_id_to_name(rbi, '')
+    puts "Building source id to symbol map"
+    source_symbols = symbols_id_to_name(source, '')
+    puts "Writing #{DIFF_RBI}"
+    diff = serialize_constants(
+      source.fetch("children", []),
+      rbi.fetch("children", []),
+      Object, false, source_symbols, rbi_symbols)
+    File.write(DIFF_RBI, diff)
+  end
+
+  def symbols_id_to_name(entry, prefix)
+    ret = {}
+    symbols_id_to_name_real(entry, prefix, ret)
+    ret
+  end
+
+  private def symbols_id_to_name_real(entry, prefix, ret)
+    name = entry["name"]["name"]
+    if prefix == '' || prefix == "<root>"
+      fqn = name.to_s
+    else
+      fqn = "#{prefix}::#{name}"
+    end
+
+    ret[entry["id"]] = fqn
+    entry.fetch("children", []).each do |child|
+      symbols_id_to_name_real(child, fqn, ret)
+    end
+  end
+
+  def serialize_constants(source, rbi, klass, is_singleton, source_symbols, rbi_symbols)
+    source_by_name = source.map {|v| [v["name"]["name"], v]}.to_h
+    ret = []
+
+    rbi.each do |rbi_entry|
+      source_entry = source_by_name[rbi_entry["name"]["name"]]
+
+      ret << serialize_alias(source_entry, rbi_entry, klass, source_symbols, rbi_symbols)
+      ret << serialize_class(source_entry, rbi_entry, klass, source_symbols, rbi_symbols, source_by_name)
+    end
+
+    ret.compact.join("\n")
+  end
+
+  def serialize_class(source_entry, rbi_entry, klass, source_symbols, rbi_symbols, source_by_name)
+    return if rbi_entry["kind"] != "CLASS"
+
+    name = rbi_entry["name"]["name"]
+    if name.start_with?('<Class:')
+      name = name.sub('<Class:', '').sub('>', '')
+      my_klass_is_singleton = true
+    else
+      my_klass_is_singleton = false
+    end
+    begin
+      my_klass = klass.const_get(name, false) # rubocop:disable PrisonGuard/NoDynamicConstAccess
+    rescue LoadError, NameError, ArgumentError => e
+      return "# #{e.message.gsub("\n", "\n# ")}"
+    end
+
+    return if !Sorbet::Private::RealStdlib.real_is_a?(my_klass, Class) && !Sorbet::Private::RealStdlib.real_is_a?(my_klass, Module)
+
+    # We specifically don't typecheck anything in T:: since it is hardcoded
+    # into sorbet
+    return if real_name(my_klass) == 'T'
+
+    source_type = nil
+    if !source_entry
+      if source_by_name[name]
+        source_type = source_by_name[name]["kind"]
+      end
+    else
+      source_type = source_entry["kind"]
+    end
+    if source_type && source_type != "CLASS"
+      return "# The source says #{real_name(my_klass)} is a #{source_type} but reflection says it is a #{rbi_entry['kind']}"
+    end
+
+    if !source_entry
+      source_children = []
+      source_mixins = []
+      is_stub = true
+    else
+      source_children = source_entry.fetch("children", [])
+      source_mixins = source_entry.fetch("mixins", [])
+      is_stub = source_entry['superClass'] && source_symbols[source_entry['superClass']] == 'Sorbet::Private::Static::StubModule'
+    end
+    rbi_children = rbi_entry.fetch("children", [])
+    rbi_mixins = rbi_entry.fetch("mixins", [])
+
+    methods = serialize_methods(source_children, rbi_children, my_klass, my_klass_is_singleton)
+    includes = serialize_includes(source_mixins, rbi_mixins, my_klass, my_klass_is_singleton, source_symbols, rbi_symbols)
+    values = serialize_values(source_children, rbi_children, my_klass, source_symbols)
+
+    ret = []
+    if !without_errors(methods).empty? || !without_errors(includes).empty? || !without_errors(values).empty? || is_stub
+      fqn = real_name(my_klass)
+      if fqn
+        klass_str = String.new
+        klass_str << (Sorbet::Private::RealStdlib.real_is_a?(my_klass, Class) ? "class #{fqn}\n" : "module #{fqn}\n")
+        klass_str << includes.join("\n")
+        klass_str << "\n" unless klass_str.end_with?("\n")
+        klass_str << methods.join("\n")
+        klass_str << "\n" unless klass_str.end_with?("\n")
+        klass_str << values.join("\n")
+        klass_str << "\n" unless klass_str.end_with?("\n")
+        klass_str << "end\n"
+        ret << klass_str
+      end
+    end
+
+    children = serialize_constants(source_children, rbi_children, my_klass, my_klass_is_singleton, source_symbols, rbi_symbols)
+    if children != ""
+      ret << children
+    end
+
+    ret.empty? ? nil : ret.join("\n")
+  end
+
+  private def without_errors(lines)
+    lines.reject {|line| line.start_with?("#")}
+  end
+
+  def serialize_alias(source_entry, rbi_entry, my_klass, source_symbols, rbi_symbols)
+    return if rbi_entry["kind"] != "STATIC_FIELD"
+    return if source_entry == rbi_entry
+    if source_entry
+      is_stub = source_entry['superClass'] && source_symbols[source_entry['superClass']] == 'Sorbet::Private::Static::StubModule'
+      if !is_stub
+        return
+      end
+    end
+    return if !rbi_entry["aliasTo"]
+
+    fqn = rbi_symbols[rbi_entry["id"]]
+    other_fqn = rbi_symbols[rbi_entry["aliasTo"]]
+    return if looks_like_stub_name(fqn)
+    ret = String.new
+    ret << "#{fqn} = #{other_fqn}\n"
+    return ret
+  end
+
+  def looks_like_stub_name(name)
+    name.include?('$')
+  end
+
+  private def serialize_values(source, rbi, klass, source_symbols)
+    source_by_name = source.map {|v| [v["name"]["name"], v]}.to_h
+    ret = []
+    rbi.each do |rbi_entry|
+      name = rbi_entry["name"]["name"]
+      source_entry = source_by_name[name]
+      if source_entry
+        is_stub = source_entry['superClass'] && source_symbols[source_entry['superClass']] == 'Sorbet::Private::Static::StubModule'
+        next unless is_stub
+      end
+      next if Sorbet::Private::ConstantLookupCache::DEPRECATED_CONSTANTS.include?("#{Sorbet::Private::RealStdlib.real_name(klass)}::#{name}")
+      begin
+        my_value = klass.const_get(name, false) # rubocop:disable PrisonGuard/NoDynamicConstAccess
+      rescue StandardError, LoadError => e
+        ret << "# #{e.message.gsub("\n", "\n# ")}"
+        next
+      end
+      next if Sorbet::Private::RealStdlib.real_is_a?(my_value, Class) || Sorbet::Private::RealStdlib.real_is_a?(my_value, Module)
+      ret << "  #{name} = ::T.let(nil, ::T.untyped)"
+    end
+    ret
+  end
+
+  # These methods are defined in C++ and we want our C++ definition to
+  # win instead of a shim.
+  BLACKLIST = Set.new([
+    [Class.object_id, "new"],
+    [BasicObject.object_id, "initialize"],
+  ]).freeze
+
+  private def serialize_methods(source, rbi, klass, is_singleton)
+    source_by_name = source.map {|v| [v["name"]["name"], v]}.to_h
+    ret = []
+    maker = Sorbet::Private::Serialize.new(constant_cache)
+    rbi.each do |rbi_entry|
+      next if rbi_entry["kind"] != "METHOD"
+      name = rbi_entry["name"]["name"]
+      next if source_by_name[name]
+
+      next if BLACKLIST.include?([klass.object_id, name])
+      next if name.start_with?('<') && name.end_with?('>')
+
+      begin
+        if is_singleton
+          method = klass.singleton_method(name)
+        else
+          method = klass.instance_method(name)
+        end
+      rescue => e
+        ret << "# #{e.message.gsub("\n", "\n# ")}"
+        next
+      end
+
+      super_method = method.super_method
+      # next if super_method && T::AbstractUtils.abstract_method?(method) == T::AbstractUtils.abstract_method?(super_method)
+
+      errors = capture_stderr do
+        ret << maker.serialize_method(method, is_singleton, with_sig: false)
+      end
+      errors.split("\n").each do |line|
+        ret << "# #{line}"
+      end
+    end
+
+    ret
+  end
+
+  private def serialize_includes(source, rbi, klass, is_singleton, source_symbols, rbi_symbols)
+    ret = []
+    source_mixins = source.map {|id| source_symbols[id]}
+    rbi_mixins = rbi.map {|id| rbi_symbols[id]}
+    rbi_mixins.each do |rbi_mixin|
+      if !source_mixins.include?(rbi_mixin)
+        keyword = is_singleton ? "extend" : "include"
+        ret << "  #{keyword} ::#{rbi_mixin}"
+      end
+    end
+    ret
+  end
+
+  def capture_stderr
+    real_stderr = $stderr
+    $stderr = StringIO.new
+    yield
+    $stderr.string
+  ensure
+    $stderr = real_stderr
+  end
+
+  private def rm_rbis
+    File.delete(HIDDEN_RBI) if File.exist?(HIDDEN_RBI)
+    File.delete(ERRORS_RBI) if File.exist?(ERRORS_RBI)
+  end
+
+  private def split_rbi
+    puts "Generating split RBIs into #{PATH}"
+    output = {
+      hidden: String.new,
+      errors: String.new,
+    }
+
+    valid = File.read(DIFF_RBI)
+    cur_output = T.let(nil, T.untyped)
+
+    valid.split("\n").each do |line|
+      category = categorize(line)
+      if category == :errors
+        # Don't ever switch to errors output permanantly
+        output[category] << line + "\n"
+        next
+      end
+      if !category.nil?
+        cur_output = output[category]
+      end
+      cur_output << line + "\n"
+    end
+
+    File.write(HIDDEN_RBI, HEADER + "\n" + output[:hidden])
+    File.write(ERRORS_RBI, HEADER + "\n" + output[:errors])
+  end
+
+  private def categorize(line)
+    if line.start_with?('#')
+      return :errors
+    end
+    return :hidden
+  end
+
+  def self.output_file
+    PATH
+  end
+end
+
+if $PROGRAM_NAME == __FILE__
+  Sorbet::Private::HiddenMethodFinder.main
+end