tapioca 0.8.3 → 0.9.0

data/lib/tapioca/gem/pipeline.rb

@@ -1,14 +1,12 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "pathname"
-
 module Tapioca
   module Gem
     class Pipeline
       extend T::Sig
       include Runtime::Reflection
-      include SignaturesHelper
+      include RBIHelper
 
       IGNORED_SYMBOLS = T.let(["YAML", "MiniTest", "Mutex"], T::Array[String])
 
@@ -25,8 +23,8 @@ module Tapioca
         @events = T.let([], T::Array[Gem::Event])
 
         @payload_symbols = T.let(Static::SymbolLoader.payload_symbols, T::Set[String])
-        @bootstrap_symbols = T.let(Static::SymbolLoader.gem_symbols(@gem).union(Static::SymbolLoader.engine_symbols),
-          T::Set[String])
+        @bootstrap_symbols = T.let(load_bootstrap_symbols(@gem), T::Set[String])
+
         @bootstrap_symbols.each { |symbol| push_symbol(symbol) }
 
         @node_listeners = T.let([], T::Array[Gem::Listeners::Base])
@@ -41,6 +39,7 @@ module Tapioca
         @node_listeners << Gem::Listeners::SorbetSignatures.new(self)
         @node_listeners << Gem::Listeners::Subconstants.new(self)
         @node_listeners << Gem::Listeners::YardDoc.new(self) if include_doc
+        @node_listeners << Gem::Listeners::ForeignConstants.new(self)
         @node_listeners << Gem::Listeners::RemoveEmptyPayloadScopes.new(self)
       end
 
@@ -60,16 +59,30 @@ module Tapioca
         @events << Gem::ConstantFound.new(symbol, constant)
       end
 
+      sig { params(symbol: String, constant: Module).void.checked(:never) }
+      def push_foreign_constant(symbol, constant)
+        @events << Gem::ForeignConstantFound.new(symbol, constant)
+      end
+
       sig { params(symbol: String, constant: Module, node: RBI::Const).void.checked(:never) }
       def push_const(symbol, constant, node)
         @events << Gem::ConstNodeAdded.new(symbol, constant, node)
       end
 
-      sig { params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never) }
+      sig do
+        params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never)
+      end
       def push_scope(symbol, constant, node)
         @events << Gem::ScopeNodeAdded.new(symbol, constant, node)
       end
 
+      sig do
+        params(symbol: String, constant: Module, node: RBI::Scope).void.checked(:never)
+      end
+      def push_foreign_scope(symbol, constant, node)
+        @events << Gem::ForeignScopeNodeAdded.new(symbol, constant, node)
+      end
+
       sig do
         params(
           symbol: String,
@@ -114,6 +127,14 @@ module Tapioca
 
       private
 
+      sig { params(gem: Gemfile::GemSpec).returns(T::Set[String]) }
+      def load_bootstrap_symbols(gem)
+        engine_symbols = Static::SymbolLoader.engine_symbols(gem)
+        gem_symbols = Static::SymbolLoader.gem_symbols(gem)
+
+        gem_symbols.union(engine_symbols)
+      end
+
       sig { returns(Gem::Event) }
       def next_event
         T.must(@events.shift)
@@ -150,13 +171,14 @@ module Tapioca
         return if name.start_with?("#<")
         return if name.downcase == name
         return if alias_namespaced?(name)
-        return if seen?(name)
 
-        constant = event.constant
-        return if T::Enum === constant # T::Enum instances are defined via `compile_enums`
+        return if T::Enum === event.constant # T::Enum instances are defined via `compile_enums`
 
-        mark_seen(name)
-        compile_constant(name, constant)
+        if event.is_a?(Gem::ForeignConstantFound)
+          compile_foreign_constant(name, event.constant)
+        else
+          compile_constant(name, event.constant)
+        end
       end
 
       sig { params(event: Gem::NodeAdded).void }
@@ -166,6 +188,11 @@ module Tapioca
 
       # Compile
 
+      sig { params(symbol: String, constant: Module).void }
+      def compile_foreign_constant(symbol, constant)
+        compile_module(symbol, constant, foreign_constant: true)
+      end
+
       sig { params(symbol: String, constant: BasicObject).void.checked(:never) }
       def compile_constant(symbol, constant)
         case constant
@@ -182,6 +209,10 @@ module Tapioca
 
       sig { params(name: String, constant: Module).void }
       def compile_alias(name, constant)
+        return if seen?(name)
+
+        mark_seen(name)
+
         return if symbol_in_payload?(name)
 
         target = name_of(constant)
@@ -199,6 +230,10 @@ module Tapioca
 
       sig { params(name: String, value: BasicObject).void.checked(:never) }
       def compile_object(name, value)
+        return if seen?(name)
+
+        mark_seen(name)
+
         return if symbol_in_payload?(name)
 
         klass = class_of(value)
@@ -228,10 +263,13 @@ module Tapioca
         @root << node
       end
 
-      sig { params(name: String, constant: Module).void }
-      def compile_module(name, constant)
-        return unless defined_in_gem?(constant, strict: false)
+      sig { params(name: String, constant: Module, foreign_constant: T::Boolean).void }
+      def compile_module(name, constant, foreign_constant: false)
+        return unless defined_in_gem?(constant, strict: false) || foreign_constant
         return if Tapioca::TypeVariableModule === constant
+        return if seen?(name)
+
+        mark_seen(name)
 
         scope =
           if constant.is_a?(Class)
@@ -241,7 +279,12 @@ module Tapioca
             RBI::Module.new(name)
           end
 
-        push_scope(name, constant, scope)
+        if foreign_constant
+          push_foreign_scope(name, constant, scope)
+        else
+          push_scope(name, constant, scope)
+        end
+
         @root << scope
       end
 

data/lib/tapioca/gem.rb

@@ -0,0 +1,6 @@
+# typed: true
+# frozen_string_literal: true
+
+require "tapioca/gem/events"
+require "tapioca/gem/listeners"
+require "tapioca/gem/pipeline"

data/lib/tapioca/gemfile.rb

@@ -1,10 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "bundler"
-require "logger"
-require "yard-sorbet"
-
 module Tapioca
   class Gemfile
     extend(T::Sig)
@@ -140,8 +136,13 @@ module Tapioca
       extend(T::Sig)
       include GemHelper
 
-      IGNORED_GEMS = T.let(["sorbet", "sorbet-static", "sorbet-runtime", "sorbet-static-and-runtime"].freeze,
-        T::Array[String])
+      IGNORED_GEMS = T.let(
+        [
+          "sorbet", "sorbet-static", "sorbet-runtime", "sorbet-static-and-runtime",
+          "debug", "fakefs",
+        ].freeze,
+        T::Array[String]
+      )
 
       sig { returns(String) }
       attr_reader :full_gem_path, :version

data/lib/tapioca/helpers/cli_helper.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "thor"
-
 module Tapioca
   module CliHelper
     extend T::Sig
@@ -30,5 +28,13 @@ module Tapioca
       rbi_formatter.max_line_length = options[:rbi_max_line_length]
       rbi_formatter
     end
+
+    sig { params(options: T::Hash[Symbol, T.untyped]).returns(T.nilable(String)) }
+    def netrc_file(options)
+      return nil if options[:auth]
+      return nil unless options[:netrc]
+
+      options[:netrc_file] || ENV["TAPIOCA_NETRC_FILE"] || File.join(ENV["HOME"].to_s, ".netrc")
+    end
   end
 end

data/lib/tapioca/helpers/config_helper.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "yaml"
-
 module Tapioca
   module ConfigHelper
     extend T::Sig

data/lib/tapioca/helpers/env_helper.rb

@@ -0,0 +1,16 @@
+# typed: true
+# frozen_string_literal: true
+
+module Tapioca
+  module EnvHelper
+    extend T::Sig
+    extend T::Helpers
+
+    requires_ancestor { Thor }
+
+    sig { params(options: T::Hash[Symbol, T.untyped]).void }
+    def set_environment(options) # rubocop:disable Naming/AccessorMethodName
+      ENV["RAILS_ENV"] = ENV["RACK_ENV"] = options[:environment]
+    end
+  end
+end

data/lib/tapioca/helpers/rbi_files_helper.rb

@@ -0,0 +1,252 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Tapioca
+  module RBIFilesHelper
+    extend T::Sig
+    extend T::Helpers
+
+    requires_ancestor { Thor::Shell }
+    requires_ancestor { SorbetHelper }
+
+    sig { params(index: RBI::Index, dir: String).void }
+    def index_payload(index, dir)
+      return unless Dir.exist?(dir)
+
+      say("Loading Sorbet payload... ")
+      files = Dir.glob("#{dir}/**/*.rbi").sort
+      parse_and_index_files(index, files)
+      say(" Done", :green)
+    end
+
+    sig { params(index: RBI::Index, kind: String, file: String).void }
+    def index_rbi(index, kind, file)
+      return unless File.exist?(file)
+
+      say("Loading #{kind} RBIs from #{file}... ")
+      parse_and_index_file(index, file)
+      say(" Done", :green)
+    end
+
+    sig { params(index: RBI::Index, kind: String, dir: String).void }
+    def index_rbis(index, kind, dir)
+      return unless Dir.exist?(dir) && !Dir.empty?(dir)
+
+      say("Loading #{kind} RBIs from #{dir}... ")
+      files = Dir.glob("#{dir}/**/*.rbi").sort
+      parse_and_index_files(index, files)
+      say(" Done", :green)
+    end
+
+    sig do
+      params(
+        index: RBI::Index,
+        shim_rbi_dir: String,
+        todo_rbi_file: String
+      ).returns(T::Hash[String, T::Array[RBI::Node]])
+    end
+    def duplicated_nodes_from_index(index, shim_rbi_dir:, todo_rbi_file:)
+      duplicates = {}
+      say("Looking for duplicates... ")
+      index.keys.each do |key|
+        nodes = index[key]
+        next unless shims_or_todos_have_duplicates?(nodes, shim_rbi_dir: shim_rbi_dir, todo_rbi_file: todo_rbi_file)
+
+        duplicates[key] = nodes
+      end
+      say(" Done", :green)
+      duplicates
+    end
+
+    sig { params(loc: RBI::Loc, path_prefix: T.nilable(String)).returns(String) }
+    def location_to_payload_url(loc, path_prefix:)
+      return loc.to_s unless path_prefix
+
+      url = loc.file || ""
+      return loc.to_s unless url.start_with?(path_prefix)
+
+      url = url.sub(path_prefix, SorbetHelper::SORBET_PAYLOAD_URL)
+      url = "#{url}#L#{loc.begin_line}"
+      url
+    end
+
+    sig do
+      params(
+        command: String,
+        gem_dir: String,
+        dsl_dir: String,
+        auto_strictness: T::Boolean,
+        gems: T::Array[Gemfile::GemSpec],
+        compilers: T::Enumerable[Class]
+      ).void
+    end
+    def validate_rbi_files(command:, gem_dir:, dsl_dir:, auto_strictness:, gems: [], compilers: [])
+      error_url_base = Spoom::Sorbet::Errors::DEFAULT_ERROR_URL_BASE
+
+      say("Checking generated RBI files... ")
+      res = sorbet(
+        "--no-config",
+        "--error-url-base=#{error_url_base}",
+        "--stop-after namer",
+        dsl_dir,
+        gem_dir
+      )
+      say(" Done", :green)
+
+      errors = Spoom::Sorbet::Errors::Parser.parse_string(res.err)
+
+      if errors.empty?
+        say("  No errors found\n\n", [:green, :bold])
+        return
+      end
+
+      parse_errors = errors.select { |error| error.code < 4000 }
+
+      if parse_errors.any?
+        say_error(<<~ERR, :red)
+
+          ##### INTERNAL ERROR #####
+
+          There are parse errors in the generated RBI files.
+
+          This seems related to a bug in Tapioca.
+          Please open an issue at https://github.com/Shopify/tapioca/issues/new with the following information:
+
+          Tapioca v#{Tapioca::VERSION}
+
+          Command:
+            #{command}
+
+        ERR
+
+        say_error(<<~ERR, :red) if gems.any?
+          Gems:
+          #{gems.map { |gem| "  #{gem.name} (#{gem.version})" }.join("\n")}
+
+        ERR
+
+        say_error(<<~ERR, :red) if compilers.any?
+          Compilers:
+          #{compilers.map { |compiler| "  #{compiler.name}" }.join("\n")}
+
+        ERR
+
+        say_error(<<~ERR, :red)
+          Errors:
+          #{parse_errors.map { |error| "  #{error}" }.join("\n")}
+
+          ##########################
+
+        ERR
+      end
+
+      if auto_strictness
+        redef_errors = errors.select { |error| error.code == 4010 }
+        update_gem_rbis_strictnesses(redef_errors, gem_dir)
+      end
+
+      Kernel.exit(1) if parse_errors.any?
+    end
+
+    private
+
+    sig { params(index: RBI::Index, files: T::Array[String]).void }
+    def parse_and_index_files(index, files)
+      files.each do |file|
+        parse_and_index_file(index, file)
+      end
+    end
+
+    sig { params(index: RBI::Index, file: String).void }
+    def parse_and_index_file(index, file)
+      tree = RBI::Parser.parse_file(file)
+      index.visit(tree)
+    rescue RBI::ParseError => e
+      say_error("\nWarning: #{e} (#{e.location})", :yellow)
+    end
+
+    sig { params(nodes: T::Array[RBI::Node], shim_rbi_dir: String, todo_rbi_file: String).returns(T::Boolean) }
+    def shims_or_todos_have_duplicates?(nodes, shim_rbi_dir:, todo_rbi_file:)
+      return false if nodes.size == 1
+
+      shims_or_todos = extract_shims_and_todos(nodes, shim_rbi_dir: shim_rbi_dir, todo_rbi_file: todo_rbi_file)
+      return false if shims_or_todos.empty?
+
+      shims_or_todos_empty_scopes = extract_empty_scopes(shims_or_todos)
+      return true unless shims_or_todos_empty_scopes.empty?
+
+      props = extract_methods_and_attrs(shims_or_todos)
+      return false if props.empty?
+
+      shims_or_todos_with_sigs = extract_nodes_with_sigs(props)
+      shims_or_todos_with_sigs.each do |shim_or_todo|
+        shims_or_todos_sigs = shim_or_todo.sigs
+
+        extract_methods_and_attrs(nodes).each do |node|
+          next if node == shim_or_todo
+          return true if shims_or_todos_sigs.all? { |sig| node.sigs.include?(sig) }
+        end
+
+        return false
+      end
+
+      true
+    end
+
+    sig { params(nodes: T::Array[RBI::Node], shim_rbi_dir: String, todo_rbi_file: String).returns(T::Array[RBI::Node]) }
+    def extract_shims_and_todos(nodes, shim_rbi_dir:, todo_rbi_file:)
+      nodes.select do |node|
+        node.loc&.file&.start_with?(shim_rbi_dir) || node.loc&.file == todo_rbi_file
+      end
+    end
+
+    sig { params(nodes: T::Array[RBI::Node]).returns(T::Array[RBI::Scope]) }
+    def extract_empty_scopes(nodes)
+      T.cast(nodes.select { |node| node.is_a?(RBI::Scope) && node.empty? }, T::Array[RBI::Scope])
+    end
+
+    sig { params(nodes: T::Array[RBI::Node]).returns(T::Array[T.any(RBI::Method, RBI::Attr)]) }
+    def extract_methods_and_attrs(nodes)
+      T.cast(nodes.select do |node|
+        node.is_a?(RBI::Method) || node.is_a?(RBI::Attr)
+      end, T::Array[T.any(RBI::Method, RBI::Attr)])
+    end
+
+    sig { params(nodes: T::Array[T.any(RBI::Method, RBI::Attr)]).returns(T::Array[T.any(RBI::Method, RBI::Attr)]) }
+    def extract_nodes_with_sigs(nodes)
+      nodes.reject { |node| node.sigs.empty? }
+    end
+
+    sig { params(errors: T::Array[Spoom::Sorbet::Errors::Error], gem_dir: String).void }
+    def update_gem_rbis_strictnesses(errors, gem_dir)
+      files = []
+
+      errors.each do |error|
+        # Collect the file with error
+        files << error.file
+        error.more.each do |line|
+          # Also collect the conflicting definition file paths
+          next unless line.include?("Previous definition")
+
+          files << line.split(":").first&.strip
+        end
+      end
+
+      files
+        .uniq
+        .sort
+        .select { |file| file.start_with?(gem_dir) }
+        .each do |file|
+          Spoom::Sorbet::Sigils.change_sigil_in_file(file, "false")
+          say("\n  Changed strictness of #{file} to `typed: false` (conflicting with DSL files)", [:yellow, :bold])
+        end
+
+      say("\n")
+    end
+
+    sig { params(path: String).returns(String) }
+    def gem_name_from_rbi_path(path)
+      T.must(File.basename(path, ".rbi").split("@").first)
+    end
+  end
+end

data/lib/tapioca/helpers/rbi_helper.rb

@@ -4,121 +4,101 @@
 module Tapioca
   module RBIHelper
     extend T::Sig
-    extend T::Helpers
+    include SorbetHelper
+    extend SorbetHelper
+    extend self
 
-    requires_ancestor { Thor::Shell }
-    requires_ancestor { SorbetHelper }
-
-    sig do
-      params(
-        command: String,
-        gem_dir: String,
-        dsl_dir: String,
-        auto_strictness: T::Boolean,
-        gems: T::Array[Gemfile::GemSpec],
-        compilers: T::Enumerable[Class]
-      ).void
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_param(name, type:)
+      create_typed_param(RBI::Param.new(name), type)
     end
-    def validate_rbi_files(command:, gem_dir:, dsl_dir:, auto_strictness:, gems: [], compilers: [])
-      error_url_base = Spoom::Sorbet::Errors::DEFAULT_ERROR_URL_BASE
-
-      say("Checking generated RBI files... ")
-      res = sorbet(
-        "--no-config",
-        "--error-url-base=#{error_url_base}",
-        "--stop-after namer",
-        dsl_dir,
-        gem_dir
-      )
-      say(" Done", :green)
-
-      errors = Spoom::Sorbet::Errors::Parser.parse_string(res.err)
-
-      if errors.empty?
-        say("  No errors found\n\n", [:green, :bold])
-        return
-      end
-
-      parse_errors = errors.select { |error| error.code < 4000 }
-
-      if parse_errors.any?
-        say_error(<<~ERR, :red)
-
-          ##### INTERNAL ERROR #####
 
-          There are parse errors in the generated RBI files.
-
-          This seems related to a bug in Tapioca.
-          Please open an issue at https://github.com/Shopify/tapioca/issues/new with the following information:
-
-          Tapioca v#{Tapioca::VERSION}
-
-          Command:
-            #{command}
-
-        ERR
-
-        say_error(<<~ERR, :red) if gems.any?
-          Gems:
-          #{gems.map { |gem| "  #{gem.name} (#{gem.version})" }.join("\n")}
+    sig { params(name: String, type: String, default: String).returns(RBI::TypedParam) }
+    def create_opt_param(name, type:, default:)
+      create_typed_param(RBI::OptParam.new(name, default), type)
+    end
 
-        ERR
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_rest_param(name, type:)
+      create_typed_param(RBI::RestParam.new(name), type)
+    end
 
-        say_error(<<~ERR, :red) if compilers.any?
-          Compilers:
-          #{compilers.map { |compiler| "  #{compiler.name}" }.join("\n")}
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_kw_param(name, type:)
+      create_typed_param(RBI::KwParam.new(name), type)
+    end
 
-        ERR
+    sig { params(name: String, type: String, default: String).returns(RBI::TypedParam) }
+    def create_kw_opt_param(name, type:, default:)
+      create_typed_param(RBI::KwOptParam.new(name, default), type)
+    end
 
-        say_error(<<~ERR, :red)
-          Errors:
-          #{parse_errors.map { |error| "  #{error}" }.join("\n")}
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_kw_rest_param(name, type:)
+      create_typed_param(RBI::KwRestParam.new(name), type)
+    end
 
-          ##########################
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_block_param(name, type:)
+      create_typed_param(RBI::BlockParam.new(name), type)
+    end
 
-        ERR
-      end
+    sig { params(param: RBI::Param, type: String).returns(RBI::TypedParam) }
+    def create_typed_param(param, type)
+      RBI::TypedParam.new(param: param, type: sanitize_signature_types(type))
+    end
 
-      if auto_strictness
-        redef_errors = errors.select { |error| error.code == 4010 }
-        update_gem_rbis_strictnesses(redef_errors, gem_dir)
-      end
+    sig { params(sig_string: String).returns(String) }
+    def sanitize_signature_types(sig_string)
+      sig_string
+        .gsub(".returns(<VOID>)", ".void")
+        .gsub("<VOID>", "void")
+        .gsub("<NOT-TYPED>", "T.untyped")
+        .gsub(".params()", "")
+    end
 
-      Kernel.exit(1) if parse_errors.any?
+    sig do
+      params(
+        type: String,
+        variance: Symbol,
+        fixed: T.nilable(String),
+        upper: T.nilable(String),
+        lower: T.nilable(String)
+      ).returns(String)
     end
+    def self.serialize_type_variable(type, variance, fixed, upper, lower)
+      variance = nil if variance == :invariant
 
-    private
+      bounds = []
+      bounds << "fixed: #{fixed}" if fixed
+      bounds << "lower: #{lower}" if lower
+      bounds << "upper: #{upper}" if upper
 
-    sig { params(errors: T::Array[Spoom::Sorbet::Errors::Error], gem_dir: String).void }
-    def update_gem_rbis_strictnesses(errors, gem_dir)
-      files = []
+      parameters = []
+      block = []
 
-      errors.each do |error|
-        # Collect the file with error
-        files << error.file
-        error.more.each do |line|
-          # Also collect the conflicting definition file paths
-          next unless line.include?("Previous definition")
+      parameters << ":#{variance}" if variance
 
-          files << line.split(":").first&.strip
-        end
+      if sorbet_supports?(:type_variable_block_syntax)
+        block = bounds
+      else
+        parameters.concat(bounds)
       end
 
-      files
-        .uniq
-        .sort
-        .select { |file| file.start_with?(gem_dir) }
-        .each do |file|
-          Spoom::Sorbet::Sigils.change_sigil_in_file(file, "false")
-          say("\n  Changed strictness of #{file} to `typed: false` (conflicting with DSL files)", [:yellow, :bold])
-        end
+      serialized = type.dup
+      serialized << "(#{parameters.join(", ")})" unless parameters.empty?
+      serialized << " { { #{block.join(", ")} } }" unless block.empty?
+      serialized
+    end
 
-      say("\n")
+    sig { params(name: String).returns(T::Boolean) }
+    def valid_method_name?(name)
+      !name.to_sym.inspect.start_with?(':"', ":@", ":$")
     end
 
-    sig { params(path: String).returns(String) }
-    def gem_name_from_rbi_path(path)
-      T.must(File.basename(path, ".rbi").split("@").first)
+    sig { params(name: String).returns(T::Boolean) }
+    def valid_parameter_name?(name)
+      /^([[:lower:]]|_|[^[[:ascii:]]])([[:alnum:]]|_|[^[[:ascii:]]])*$/.match?(name)
     end
   end
 end