tapioca 0.8.3 → 0.9.2

data/lib/tapioca/gem/listeners/sorbet_props.rb

@@ -27,6 +27,11 @@ module Tapioca
             end
           end
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners/sorbet_required_ancestors.rb

@@ -18,6 +18,11 @@ module Tapioca
             event.node << RBI::RequiresAncestor.new(ancestor.to_s)
           end
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners/sorbet_signatures.rb

@@ -8,7 +8,7 @@ module Tapioca
         extend T::Sig
 
         include Runtime::Reflection
-        include SignaturesHelper
+        include RBIHelper
 
         TYPE_PARAMETER_MATCHER = /T\.type_parameter\(:?([[:word:]]+)\)/
 
@@ -73,6 +73,11 @@ module Tapioca
 
           final_methods.include?(signature.method_name)
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners/sorbet_type_variables.rb

@@ -45,6 +45,11 @@ module Tapioca
 
           tree << RBI::Extend.new("T::Generic")
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners/source_location.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Tapioca
+  module Gem
+    module Listeners
+      class SourceLocation < Base
+        extend T::Sig
+
+        private
+
+        sig { override.params(event: ConstNodeAdded).void }
+        def on_const(event)
+          file, line = Object.const_source_location(event.symbol)
+          add_source_location_comment(event.node, file, line)
+        end
+
+        sig { override.params(event: ScopeNodeAdded).void }
+        def on_scope(event)
+          # Instead of using `const_source_location`, which always reports the first place where a constant is defined,
+          # we filter the locations tracked by ConstantDefinition. This allows us to provide the correct location for
+          # constants that are defined by multiple gems.
+          locations = Runtime::Trackers::ConstantDefinition.locations_for(event.constant)
+          location = locations.find do |loc|
+            Pathname.new(loc.path).realpath.to_s.include?(@pipeline.gem.full_gem_path)
+          end
+
+          # The location may still be nil in some situations, like constant aliases (e.g.: MyAlias = OtherConst). These
+          # are quite difficult to attribute a correct location, given that the source location points to the original
+          # constants and not the alias
+          add_source_location_comment(event.node, location.path, location.lineno) unless location.nil?
+        end
+
+        sig { override.params(event: MethodNodeAdded).void }
+        def on_method(event)
+          file, line = event.method.source_location
+          add_source_location_comment(event.node, file, line)
+        end
+
+        sig { params(node: RBI::NodeWithComments, file: T.nilable(String), line: T.nilable(Integer)).void }
+        def add_source_location_comment(node, file, line)
+          return unless file && line
+
+          gem = @pipeline.gem
+          path = Pathname.new(file)
+          return unless File.exist?(path)
+
+          # On native extensions, the source location may point to a shared object (.so, .bundle) file, which we cannot
+          # use for jump to definition. Only add source comments on Ruby files
+          return unless path.extname == ".rb"
+
+          path = if path.realpath.to_s.start_with?(gem.full_gem_path)
+            "#{gem.name}-#{gem.version}/#{path.realpath.relative_path_from(gem.full_gem_path)}"
+          else
+            path.sub("#{Bundler.bundle_path}/gems/", "").to_s
+          end
+
+          # Strip out the RUBY_ROOT prefix, which is different for each user
+          path = path.sub(RbConfig::CONFIG["rubylibdir"], "RUBY_ROOT")
+
+          node.comments << RBI::Comment.new("") if node.comments.any?
+          node.comments << RBI::Comment.new("source://#{path}:#{line}")
+        end
+      end
+    end
+  end
+end

data/lib/tapioca/gem/listeners/subconstants.rb

@@ -31,6 +31,11 @@ module Tapioca
             @pipeline.push_constant(name, subconstant)
           end
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners/yard_doc.rb

@@ -90,6 +90,11 @@ module Tapioca
 
           comments
         end
+
+        sig { override.params(event: NodeAdded).returns(T::Boolean) }
+        def ignore?(event)
+          event.is_a?(Tapioca::Gem::ForeignScopeNodeAdded)
+        end
       end
     end
   end

data/lib/tapioca/gem/listeners.rb

@@ -13,4 +13,6 @@ require "tapioca/gem/listeners/sorbet_required_ancestors"
 require "tapioca/gem/listeners/sorbet_signatures"
 require "tapioca/gem/listeners/sorbet_type_variables"
 require "tapioca/gem/listeners/subconstants"
+require "tapioca/gem/listeners/foreign_constants"
 require "tapioca/gem/listeners/yard_doc"
+require "tapioca/gem/listeners/source_location"

data/lib/tapioca/gem/pipeline.rb

@@ -1,22 +1,20 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "pathname"
-
 module Tapioca
   module Gem
     class Pipeline
       extend T::Sig
       include Runtime::Reflection
-      include SignaturesHelper
+      include RBIHelper
 
       IGNORED_SYMBOLS = T.let(["YAML", "MiniTest", "Mutex"], T::Array[String])
 
       sig { returns(Gemfile::GemSpec) }
       attr_reader :gem
 
-      sig { params(gem: Gemfile::GemSpec, include_doc: T::Boolean).void }
-      def initialize(gem, include_doc: false)
+      sig { params(gem: Gemfile::GemSpec, include_doc: T::Boolean, include_loc: T::Boolean).void }
+      def initialize(gem, include_doc: false, include_loc: false)
         @root = T.let(RBI::Tree.new, RBI::Tree)
         @gem = gem
         @seen = T.let(Set.new, T::Set[String])
@@ -25,8 +23,8 @@ module Tapioca
         @events = T.let([], T::Array[Gem::Event])
 
         @payload_symbols = T.let(Static::SymbolLoader.payload_symbols, T::Set[String])
-        @bootstrap_symbols = T.let(Static::SymbolLoader.gem_symbols(@gem).union(Static::SymbolLoader.engine_symbols),
-          T::Set[String])
+        @bootstrap_symbols = T.let(load_bootstrap_symbols(@gem), T::Set[String])
+
         @bootstrap_symbols.each { |symbol| push_symbol(symbol) }
 
         @node_listeners = T.let([], T::Array[Gem::Listeners::Base])
@@ -41,6 +39,8 @@ module Tapioca
         @node_listeners << Gem::Listeners::SorbetSignatures.new(self)
         @node_listeners << Gem::Listeners::Subconstants.new(self)
         @node_listeners << Gem::Listeners::YardDoc.new(self) if include_doc
+        @node_listeners << Gem::Listeners::ForeignConstants.new(self)
+        @node_listeners << Gem::Listeners::SourceLocation.new(self) if include_loc
         @node_listeners << Gem::Listeners::RemoveEmptyPayloadScopes.new(self)
       end
 
@@ -60,27 +60,42 @@ module Tapioca
         @events << Gem::ConstantFound.new(symbol, constant)
       end
 
+      sig { params(symbol: String, constant: Module).void.checked(:never) }
+      def push_foreign_constant(symbol, constant)
+        @events << Gem::ForeignConstantFound.new(symbol, constant)
+      end
+
       sig { params(symbol: String, constant: Module, node: RBI::Const).void.checked(:never) }
       def push_const(symbol, constant, node)
         @events << Gem::ConstNodeAdded.new(symbol, constant, node)
       end
 
-      sig { params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never) }
+      sig do
+        params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never)
+      end
       def push_scope(symbol, constant, node)
         @events << Gem::ScopeNodeAdded.new(symbol, constant, node)
       end
 
+      sig do
+        params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never)
+      end
+      def push_foreign_scope(symbol, constant, node)
+        @events << Gem::ForeignScopeNodeAdded.new(symbol, constant, node)
+      end
+
       sig do
         params(
           symbol: String,
           constant: Module,
+          method: UnboundMethod,
           node: RBI::Method,
           signature: T.untyped,
           parameters: T::Array[[Symbol, String]]
         ).void.checked(:never)
       end
-      def push_method(symbol, constant, node, signature, parameters)
-        @events << Gem::MethodNodeAdded.new(symbol, constant, node, signature, parameters)
+      def push_method(symbol, constant, method, node, signature, parameters) # rubocop:disable Metrics/ParameterLists
+        @events << Gem::MethodNodeAdded.new(symbol, constant, method, node, signature, parameters)
       end
 
       sig { params(symbol_name: String).returns(T::Boolean) }
@@ -114,6 +129,14 @@ module Tapioca
 
       private
 
+      sig { params(gem: Gemfile::GemSpec).returns(T::Set[String]) }
+      def load_bootstrap_symbols(gem)
+        engine_symbols = Static::SymbolLoader.engine_symbols(gem)
+        gem_symbols = Static::SymbolLoader.gem_symbols(gem)
+
+        gem_symbols.union(engine_symbols)
+      end
+
       sig { returns(Gem::Event) }
       def next_event
         T.must(@events.shift)
@@ -150,13 +173,14 @@ module Tapioca
         return if name.start_with?("#<")
         return if name.downcase == name
         return if alias_namespaced?(name)
-        return if seen?(name)
 
-        constant = event.constant
-        return if T::Enum === constant # T::Enum instances are defined via `compile_enums`
+        return if T::Enum === event.constant # T::Enum instances are defined via `compile_enums`
 
-        mark_seen(name)
-        compile_constant(name, constant)
+        if event.is_a?(Gem::ForeignConstantFound)
+          compile_foreign_constant(name, event.constant)
+        else
+          compile_constant(name, event.constant)
+        end
       end
 
       sig { params(event: Gem::NodeAdded).void }
@@ -166,6 +190,11 @@ module Tapioca
 
       # Compile
 
+      sig { params(symbol: String, constant: Module).void }
+      def compile_foreign_constant(symbol, constant)
+        compile_module(symbol, constant, foreign_constant: true)
+      end
+
       sig { params(symbol: String, constant: BasicObject).void.checked(:never) }
       def compile_constant(symbol, constant)
         case constant
@@ -182,6 +211,10 @@ module Tapioca
 
       sig { params(name: String, constant: Module).void }
       def compile_alias(name, constant)
+        return if seen?(name)
+
+        mark_seen(name)
+
         return if symbol_in_payload?(name)
 
         target = name_of(constant)
@@ -199,6 +232,10 @@ module Tapioca
 
       sig { params(name: String, value: BasicObject).void.checked(:never) }
       def compile_object(name, value)
+        return if seen?(name)
+
+        mark_seen(name)
+
         return if symbol_in_payload?(name)
 
         klass = class_of(value)
@@ -228,10 +265,13 @@ module Tapioca
         @root << node
       end
 
-      sig { params(name: String, constant: Module).void }
-      def compile_module(name, constant)
-        return unless defined_in_gem?(constant, strict: false)
+      sig { params(name: String, constant: Module, foreign_constant: T::Boolean).void }
+      def compile_module(name, constant, foreign_constant: false)
+        return unless defined_in_gem?(constant, strict: false) || foreign_constant
         return if Tapioca::TypeVariableModule === constant
+        return if seen?(name)
+
+        mark_seen(name)
 
         scope =
           if constant.is_a?(Class)
@@ -241,7 +281,12 @@ module Tapioca
             RBI::Module.new(name)
           end
 
-        push_scope(name, constant, scope)
+        if foreign_constant
+          push_foreign_scope(name, constant, scope)
+        else
+          push_scope(name, constant, scope)
+        end
+
         @root << scope
       end
 

data/lib/tapioca/gem.rb

@@ -0,0 +1,6 @@
+# typed: true
+# frozen_string_literal: true
+
+require "tapioca/gem/events"
+require "tapioca/gem/listeners"
+require "tapioca/gem/pipeline"

data/lib/tapioca/gemfile.rb

@@ -1,10 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "bundler"
-require "logger"
-require "yard-sorbet"
-
 module Tapioca
   class Gemfile
     extend(T::Sig)
@@ -140,8 +136,13 @@ module Tapioca
       extend(T::Sig)
       include GemHelper
 
-      IGNORED_GEMS = T.let(["sorbet", "sorbet-static", "sorbet-runtime", "sorbet-static-and-runtime"].freeze,
-        T::Array[String])
+      IGNORED_GEMS = T.let(
+        [
+          "sorbet", "sorbet-static", "sorbet-runtime", "sorbet-static-and-runtime",
+          "debug", "fakefs",
+        ].freeze,
+        T::Array[String]
+      )
 
       sig { returns(String) }
       attr_reader :full_gem_path, :version

data/lib/tapioca/helpers/cli_helper.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "thor"
-
 module Tapioca
   module CliHelper
     extend T::Sig
@@ -30,5 +28,13 @@ module Tapioca
       rbi_formatter.max_line_length = options[:rbi_max_line_length]
       rbi_formatter
     end
+
+    sig { params(options: T::Hash[Symbol, T.untyped]).returns(T.nilable(String)) }
+    def netrc_file(options)
+      return nil if options[:auth]
+      return nil unless options[:netrc]
+
+      options[:netrc_file] || ENV["TAPIOCA_NETRC_FILE"] || File.join(ENV["HOME"].to_s, ".netrc")
+    end
   end
 end

data/lib/tapioca/helpers/config_helper.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "yaml"
-
 module Tapioca
   module ConfigHelper
     extend T::Sig

data/lib/tapioca/helpers/env_helper.rb

@@ -0,0 +1,16 @@
+# typed: true
+# frozen_string_literal: true
+
+module Tapioca
+  module EnvHelper
+    extend T::Sig
+    extend T::Helpers
+
+    requires_ancestor { Thor }
+
+    sig { params(options: T::Hash[Symbol, T.untyped]).void }
+    def set_environment(options) # rubocop:disable Naming/AccessorMethodName
+      ENV["RAILS_ENV"] = ENV["RACK_ENV"] = options[:environment]
+    end
+  end
+end

data/lib/tapioca/helpers/rbi_files_helper.rb

@@ -0,0 +1,255 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Tapioca
+  module RBIFilesHelper
+    extend T::Sig
+    extend T::Helpers
+
+    requires_ancestor { Thor::Shell }
+    requires_ancestor { SorbetHelper }
+
+    sig { params(index: RBI::Index, kind: String, file: String).void }
+    def index_rbi(index, kind, file)
+      return unless File.exist?(file)
+
+      say("Loading #{kind} RBIs from #{file}... ")
+      time = Benchmark.realtime do
+        parse_and_index_files(index, [file], number_of_workers: 1)
+      end
+      say(" Done ", :green)
+      say("(#{time.round(2)}s)")
+    end
+
+    sig { params(index: RBI::Index, kind: String, dir: String, number_of_workers: T.nilable(Integer)).void }
+    def index_rbis(index, kind, dir, number_of_workers:)
+      return unless Dir.exist?(dir) && !Dir.empty?(dir)
+
+      if kind == "payload"
+        say("Loading Sorbet payload... ")
+      else
+        say("Loading #{kind} RBIs from #{dir}... ")
+      end
+      time = Benchmark.realtime do
+        files = Dir.glob("#{dir}/**/*.rbi").sort
+        parse_and_index_files(index, files, number_of_workers: number_of_workers)
+      end
+      say(" Done ", :green)
+      say("(#{time.round(2)}s)")
+    end
+
+    sig do
+      params(
+        index: RBI::Index,
+        shim_rbi_dir: String,
+        todo_rbi_file: String
+      ).returns(T::Hash[String, T::Array[RBI::Node]])
+    end
+    def duplicated_nodes_from_index(index, shim_rbi_dir:, todo_rbi_file:)
+      duplicates = {}
+      say("Looking for duplicates... ")
+      time = Benchmark.realtime do
+        index.keys.each do |key|
+          nodes = index[key]
+          next unless shims_or_todos_have_duplicates?(nodes, shim_rbi_dir: shim_rbi_dir, todo_rbi_file: todo_rbi_file)
+
+          duplicates[key] = nodes
+        end
+      end
+      say(" Done ", :green)
+      say("(#{time.round(2)}s)")
+      duplicates
+    end
+
+    sig { params(loc: RBI::Loc, path_prefix: T.nilable(String)).returns(String) }
+    def location_to_payload_url(loc, path_prefix:)
+      return loc.to_s unless path_prefix
+
+      url = loc.file || ""
+      return loc.to_s unless url.start_with?(path_prefix)
+
+      url = url.sub(path_prefix, SorbetHelper::SORBET_PAYLOAD_URL)
+      url = "#{url}#L#{loc.begin_line}"
+      url
+    end
+
+    sig do
+      params(
+        command: String,
+        gem_dir: String,
+        dsl_dir: String,
+        auto_strictness: T::Boolean,
+        gems: T::Array[Gemfile::GemSpec],
+        compilers: T::Enumerable[Class]
+      ).void
+    end
+    def validate_rbi_files(command:, gem_dir:, dsl_dir:, auto_strictness:, gems: [], compilers: [])
+      error_url_base = Spoom::Sorbet::Errors::DEFAULT_ERROR_URL_BASE
+
+      say("Checking generated RBI files... ")
+      res = sorbet(
+        "--no-config",
+        "--error-url-base=#{error_url_base}",
+        "--stop-after namer",
+        dsl_dir,
+        gem_dir
+      )
+      say(" Done", :green)
+
+      errors = Spoom::Sorbet::Errors::Parser.parse_string(res.err)
+
+      if errors.empty?
+        say("  No errors found\n\n", [:green, :bold])
+        return
+      end
+
+      parse_errors = errors.select { |error| error.code < 4000 }
+
+      if parse_errors.any?
+        say_error(<<~ERR, :red)
+
+          ##### INTERNAL ERROR #####
+
+          There are parse errors in the generated RBI files.
+
+          This seems related to a bug in Tapioca.
+          Please open an issue at https://github.com/Shopify/tapioca/issues/new with the following information:
+
+          Tapioca v#{Tapioca::VERSION}
+
+          Command:
+            #{command}
+
+        ERR
+
+        say_error(<<~ERR, :red) if gems.any?
+          Gems:
+          #{gems.map { |gem| "  #{gem.name} (#{gem.version})" }.join("\n")}
+
+        ERR
+
+        say_error(<<~ERR, :red) if compilers.any?
+          Compilers:
+          #{compilers.map { |compiler| "  #{compiler.name}" }.join("\n")}
+
+        ERR
+
+        say_error(<<~ERR, :red)
+          Errors:
+          #{parse_errors.map { |error| "  #{error}" }.join("\n")}
+
+          ##########################
+
+        ERR
+      end
+
+      if auto_strictness
+        redef_errors = errors.select { |error| error.code == 4010 }
+        update_gem_rbis_strictnesses(redef_errors, gem_dir)
+      end
+
+      Kernel.exit(1) if parse_errors.any?
+    end
+
+    private
+
+    sig { params(index: RBI::Index, files: T::Array[String], number_of_workers: T.nilable(Integer)).void }
+    def parse_and_index_files(index, files, number_of_workers:)
+      executor = Executor.new(files, number_of_workers: number_of_workers)
+
+      trees = executor.run_in_parallel do |file|
+        next if Spoom::Sorbet::Sigils.file_strictness(file) == "ignore"
+
+        RBI::Parser.parse_file(file)
+      rescue RBI::ParseError => e
+        say_error("\nWarning: #{e} (#{e.location})", :yellow)
+      end
+
+      index.visit_all(trees)
+    end
+
+    sig { params(nodes: T::Array[RBI::Node], shim_rbi_dir: String, todo_rbi_file: String).returns(T::Boolean) }
+    def shims_or_todos_have_duplicates?(nodes, shim_rbi_dir:, todo_rbi_file:)
+      return false if nodes.size == 1
+
+      shims_or_todos = extract_shims_and_todos(nodes, shim_rbi_dir: shim_rbi_dir, todo_rbi_file: todo_rbi_file)
+      return false if shims_or_todos.empty?
+
+      shims_or_todos_empty_scopes = extract_empty_scopes(shims_or_todos)
+      return true unless shims_or_todos_empty_scopes.empty?
+
+      props = extract_methods_and_attrs(shims_or_todos)
+      return false if props.empty?
+
+      shims_or_todos_with_sigs = extract_nodes_with_sigs(props)
+      shims_or_todos_with_sigs.each do |shim_or_todo|
+        shims_or_todos_sigs = shim_or_todo.sigs
+
+        extract_methods_and_attrs(nodes).each do |node|
+          next if node == shim_or_todo
+          return true if shims_or_todos_sigs.all? { |sig| node.sigs.include?(sig) }
+        end
+
+        return false
+      end
+
+      true
+    end
+
+    sig { params(nodes: T::Array[RBI::Node], shim_rbi_dir: String, todo_rbi_file: String).returns(T::Array[RBI::Node]) }
+    def extract_shims_and_todos(nodes, shim_rbi_dir:, todo_rbi_file:)
+      nodes.select do |node|
+        node.loc&.file&.start_with?(shim_rbi_dir) || node.loc&.file == todo_rbi_file
+      end
+    end
+
+    sig { params(nodes: T::Array[RBI::Node]).returns(T::Array[RBI::Scope]) }
+    def extract_empty_scopes(nodes)
+      T.cast(nodes.select { |node| node.is_a?(RBI::Scope) && node.empty? }, T::Array[RBI::Scope])
+    end
+
+    sig { params(nodes: T::Array[RBI::Node]).returns(T::Array[T.any(RBI::Method, RBI::Attr)]) }
+    def extract_methods_and_attrs(nodes)
+      T.cast(nodes.select do |node|
+        node.is_a?(RBI::Method) || node.is_a?(RBI::Attr)
+      end, T::Array[T.any(RBI::Method, RBI::Attr)])
+    end
+
+    sig { params(nodes: T::Array[T.any(RBI::Method, RBI::Attr)]).returns(T::Array[T.any(RBI::Method, RBI::Attr)]) }
+    def extract_nodes_with_sigs(nodes)
+      nodes.reject { |node| node.sigs.empty? }
+    end
+
+    sig { params(errors: T::Array[Spoom::Sorbet::Errors::Error], gem_dir: String).void }
+    def update_gem_rbis_strictnesses(errors, gem_dir)
+      files = []
+
+      errors.each do |error|
+        # Collect the file with error
+        files << error.file
+        error.more.each do |line|
+          # Also collect the conflicting definition file paths
+          next unless line.include?("Previous definition")
+
+          files << line.split(":").first&.strip
+        end
+      end
+
+      files
+        .uniq
+        .sort
+        .select { |file| file.start_with?(gem_dir) }
+        .each do |file|
+          Spoom::Sorbet::Sigils.change_sigil_in_file(file, "false")
+          say("\n  Changed strictness of #{file} to `typed: false` (conflicting with DSL files)", [:yellow, :bold])
+        end
+
+      say("\n")
+    end
+
+    sig { params(path: String).returns(String) }
+    def gem_name_from_rbi_path(path)
+      T.must(File.basename(path, ".rbi").split("@").first)
+    end
+  end
+end