tapioca 0.8.3 → 0.9.2

data/lib/tapioca/helpers/rbi_helper.rb

@@ -4,121 +4,125 @@
 module Tapioca
   module RBIHelper
     extend T::Sig
-    extend T::Helpers
+    include SorbetHelper
+    extend SorbetHelper
+    extend self
 
-    requires_ancestor { Thor::Shell }
-    requires_ancestor { SorbetHelper }
-
-    sig do
-      params(
-        command: String,
-        gem_dir: String,
-        dsl_dir: String,
-        auto_strictness: T::Boolean,
-        gems: T::Array[Gemfile::GemSpec],
-        compilers: T::Enumerable[Class]
-      ).void
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_param(name, type:)
+      create_typed_param(RBI::Param.new(name), type)
     end
-    def validate_rbi_files(command:, gem_dir:, dsl_dir:, auto_strictness:, gems: [], compilers: [])
-      error_url_base = Spoom::Sorbet::Errors::DEFAULT_ERROR_URL_BASE
-
-      say("Checking generated RBI files... ")
-      res = sorbet(
-        "--no-config",
-        "--error-url-base=#{error_url_base}",
-        "--stop-after namer",
-        dsl_dir,
-        gem_dir
-      )
-      say(" Done", :green)
-
-      errors = Spoom::Sorbet::Errors::Parser.parse_string(res.err)
-
-      if errors.empty?
-        say("  No errors found\n\n", [:green, :bold])
-        return
-      end
-
-      parse_errors = errors.select { |error| error.code < 4000 }
-
-      if parse_errors.any?
-        say_error(<<~ERR, :red)
-
-          ##### INTERNAL ERROR #####
 
-          There are parse errors in the generated RBI files.
-
-          This seems related to a bug in Tapioca.
-          Please open an issue at https://github.com/Shopify/tapioca/issues/new with the following information:
-
-          Tapioca v#{Tapioca::VERSION}
-
-          Command:
-            #{command}
-
-        ERR
-
-        say_error(<<~ERR, :red) if gems.any?
-          Gems:
-          #{gems.map { |gem| "  #{gem.name} (#{gem.version})" }.join("\n")}
+    sig { params(name: String, type: String, default: String).returns(RBI::TypedParam) }
+    def create_opt_param(name, type:, default:)
+      create_typed_param(RBI::OptParam.new(name, default), type)
+    end
 
-        ERR
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_rest_param(name, type:)
+      create_typed_param(RBI::RestParam.new(name), type)
+    end
 
-        say_error(<<~ERR, :red) if compilers.any?
-          Compilers:
-          #{compilers.map { |compiler| "  #{compiler.name}" }.join("\n")}
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_kw_param(name, type:)
+      create_typed_param(RBI::KwParam.new(name), type)
+    end
 
-        ERR
+    sig { params(name: String, type: String, default: String).returns(RBI::TypedParam) }
+    def create_kw_opt_param(name, type:, default:)
+      create_typed_param(RBI::KwOptParam.new(name, default), type)
+    end
 
-        say_error(<<~ERR, :red)
-          Errors:
-          #{parse_errors.map { |error| "  #{error}" }.join("\n")}
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_kw_rest_param(name, type:)
+      create_typed_param(RBI::KwRestParam.new(name), type)
+    end
 
-          ##########################
+    sig { params(name: String, type: String).returns(RBI::TypedParam) }
+    def create_block_param(name, type:)
+      create_typed_param(RBI::BlockParam.new(name), type)
+    end
 
-        ERR
-      end
+    sig { params(param: RBI::Param, type: String).returns(RBI::TypedParam) }
+    def create_typed_param(param, type)
+      RBI::TypedParam.new(param: param, type: sanitize_signature_types(type))
+    end
 
-      if auto_strictness
-        redef_errors = errors.select { |error| error.code == 4010 }
-        update_gem_rbis_strictnesses(redef_errors, gem_dir)
-      end
+    sig { params(sig_string: String).returns(String) }
+    def sanitize_signature_types(sig_string)
+      sig_string
+        .gsub(".returns(<VOID>)", ".void")
+        .gsub("<VOID>", "void")
+        .gsub("<NOT-TYPED>", "T.untyped")
+        .gsub(".params()", "")
+    end
 
-      Kernel.exit(1) if parse_errors.any?
+    sig do
+      params(
+        type: String,
+        variance: Symbol,
+        fixed: T.nilable(String),
+        upper: T.nilable(String),
+        lower: T.nilable(String)
+      ).returns(String)
     end
+    def self.serialize_type_variable(type, variance, fixed, upper, lower)
+      variance = nil if variance == :invariant
 
-    private
+      bounds = []
+      bounds << "fixed: #{fixed}" if fixed
+      bounds << "lower: #{lower}" if lower
+      bounds << "upper: #{upper}" if upper
 
-    sig { params(errors: T::Array[Spoom::Sorbet::Errors::Error], gem_dir: String).void }
-    def update_gem_rbis_strictnesses(errors, gem_dir)
-      files = []
+      parameters = []
+      block = []
 
-      errors.each do |error|
-        # Collect the file with error
-        files << error.file
-        error.more.each do |line|
-          # Also collect the conflicting definition file paths
-          next unless line.include?("Previous definition")
+      parameters << ":#{variance}" if variance
 
-          files << line.split(":").first&.strip
-        end
+      if sorbet_supports?(:type_variable_block_syntax)
+        block = bounds
+      else
+        parameters.concat(bounds)
       end
 
-      files
-        .uniq
-        .sort
-        .select { |file| file.start_with?(gem_dir) }
-        .each do |file|
-          Spoom::Sorbet::Sigils.change_sigil_in_file(file, "false")
-          say("\n  Changed strictness of #{file} to `typed: false` (conflicting with DSL files)", [:yellow, :bold])
-        end
+      serialized = type.dup
+      serialized << "(#{parameters.join(", ")})" unless parameters.empty?
+      serialized << " { { #{block.join(", ")} } }" unless block.empty?
+      serialized
+    end
 
-      say("\n")
+    sig { params(name: String).returns(T::Boolean) }
+    def valid_method_name?(name)
+      # try to parse a method definition with this name
+      iseq = RubyVM::InstructionSequence.compile("def #{name}; end", nil, nil, 0, false)
+      # pull out the first operation in the instruction sequence and its first argument
+      op, arg, _data = iseq.to_a.dig(-1, 0)
+      # make sure that the operation is a method definition and the method that was
+      # defined has the expected name, for example, for `def !foo; end` we don't get
+      # a syntax error but instead get a method defined as `"foo"`
+      op == :definemethod && arg == name.to_sym
+    rescue SyntaxError
+      false
     end
 
-    sig { params(path: String).returns(String) }
-    def gem_name_from_rbi_path(path)
-      T.must(File.basename(path, ".rbi").split("@").first)
+    sig { params(name: String).returns(T::Boolean) }
+    def valid_parameter_name?(name)
+      sentinel_method_name = :sentinel_method_name
+      # try to parse a method definition with this name as the name of a
+      # keyword parameter. If we use a positional parameter, then parameter names
+      # like `&` (and maybe others) will be treated like `def foo(&); end` and will
+      # thus be considered valid. Using a required keyword parameter prevents that
+      # confusion between Ruby syntax and parameter name.
+      iseq = RubyVM::InstructionSequence.compile("def #{sentinel_method_name}(#{name}:); end", nil, nil, 0, false)
+      # pull out the first operation in the instruction sequence and its first argument and data
+      op, arg, data = iseq.to_a.dig(-1, 0)
+      # make sure that:
+      # 1. a method was defined, and
+      # 2. the method has the expected method name, and
+      # 3. the method has a keyword parameter with the expected name
+      op == :definemethod && arg == sentinel_method_name && data.dig(11, :keyword, 0) == name.to_sym
+    rescue SyntaxError
+      false
     end
   end
 end

data/lib/tapioca/helpers/sorbet_helper.rb

@@ -1,9 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "pathname"
-require "shellwords"
-
 module Tapioca
   module SorbetHelper
     extend T::Sig
@@ -20,6 +17,8 @@ module Tapioca
 
     SORBET_EXE_PATH_ENV_VAR = "TAPIOCA_SORBET_EXE"
 
+    SORBET_PAYLOAD_URL = "https://github.com/sorbet/sorbet/tree/master/rbi"
+
     FEATURE_REQUIREMENTS = T.let({
       to_ary_nil_support: ::Gem::Requirement.new(">= 0.5.9220"),         # https://github.com/sorbet/sorbet/pull/4706
       print_payload_sources: ::Gem::Requirement.new(">= 0.5.9818"),      # https://github.com/sorbet/sorbet/pull/5504

data/lib/tapioca/helpers/test/content.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "fileutils"
-
 module Tapioca
   module Helpers
     module Test

data/lib/tapioca/helpers/test/template.rb

@@ -1,8 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-require "erb"
-
 module Tapioca
   module Helpers
     module Test

data/lib/tapioca/internal.rb

@@ -1,32 +1,56 @@
 # typed: strict
 # frozen_string_literal: true
 
+require "set"
+
 require "tapioca"
 require "tapioca/runtime/reflection"
 require "tapioca/runtime/trackers"
+
+require "benchmark"
+require "bundler"
+require "erb"
+require "etc"
+require "fileutils"
+require "json"
+require "logger"
+require "net/http"
+require "netrc"
+require "parallel"
+require "pathname"
+require "shellwords"
+require "spoom"
+require "tempfile"
+require "thor"
+require "yaml"
+require "yard-sorbet"
+
 require "tapioca/runtime/dynamic_mixin_compiler"
 require "tapioca/helpers/gem_helper"
 require "tapioca/runtime/loader"
+
 require "tapioca/helpers/sorbet_helper"
-require "tapioca/helpers/type_variable_helper"
-require "tapioca/sorbet_ext/generic_name_patch"
+require "tapioca/helpers/rbi_helper"
 require "tapioca/sorbet_ext/fixed_hash_patch"
+require "tapioca/sorbet_ext/name_patch"
+require "tapioca/sorbet_ext/generic_name_patch"
+require "tapioca/sorbet_ext/proc_bind_patch"
 require "tapioca/runtime/generic_type_registry"
+
 require "tapioca/helpers/cli_helper"
 require "tapioca/helpers/config_helper"
-require "tapioca/helpers/signatures_helper"
-require "tapioca/helpers/rbi_helper"
-require "tapioca/helpers/shims_helper"
+require "tapioca/helpers/rbi_files_helper"
+require "tapioca/helpers/env_helper"
+
 require "tapioca/repo_index"
-require "tapioca/commands"
-require "tapioca/cli"
 require "tapioca/gemfile"
 require "tapioca/executor"
+
 require "tapioca/static/symbol_table_parser"
 require "tapioca/static/symbol_loader"
-require "tapioca/gem/events"
-require "tapioca/gem/listeners"
-require "tapioca/gem/pipeline"
-require "tapioca/dsl/compiler"
-require "tapioca/dsl/pipeline"
 require "tapioca/static/requires_compiler"
+
+require "tapioca/gem"
+require "tapioca/dsl"
+require "tapioca/commands"
+require "tapioca/cli"

data/lib/tapioca/rbi_ext/model.rb

@@ -72,7 +72,7 @@ module RBI
       ).void
     end
     def create_type_variable(name, type:, variance: :invariant, fixed: nil, upper: nil, lower: nil)
-      value = Tapioca::TypeVariableHelper.serialize_type_variable(type, variance, fixed, upper, lower)
+      value = Tapioca::RBIHelper.serialize_type_variable(type, variance, fixed, upper, lower)
       create_node(RBI::TypeMember.new(name, value))
     end
 
@@ -86,7 +86,7 @@ module RBI
       ).void
     end
     def create_method(name, parameters: [], return_type: "T.untyped", class_method: false, visibility: RBI::Public.new)
-      return unless valid_method_name?(name)
+      return unless Tapioca::RBIHelper.valid_method_name?(name)
 
       sig = RBI::Sig.new(return_type: return_type)
       method = RBI::Method.new(name, sigs: [sig], is_singleton: class_method, visibility: visibility)
@@ -99,19 +99,6 @@ module RBI
 
     private
 
-    SPECIAL_METHOD_NAMES = T.let(
-      ["!", "~", "+@", "**", "-@", "*", "/", "%", "+", "-", "<<", ">>", "&", "|", "^", "<", "<=", "=>", ">", ">=",
-       "==", "===", "!=", "=~", "!~", "<=>", "[]", "[]=", "`",].freeze,
-      T::Array[String]
-    )
-
-    sig { params(name: String).returns(T::Boolean) }
-    def valid_method_name?(name)
-      return true if SPECIAL_METHOD_NAMES.include?(name)
-
-      !!name.match(/^[a-zA-Z_][[:word:]]*[?!=]?$/)
-    end
-
     sig { returns(T::Hash[String, RBI::Node]) }
     def nodes_cache
       T.must(@nodes_cache ||= T.let({}, T.nilable(T::Hash[String, Node])))

data/lib/tapioca/runtime/dynamic_mixin_compiler.rb

@@ -35,22 +35,24 @@ module Tapioca
             # before the actual include
             before = singleton_class.ancestors
             # Call the actual `include` method with the supplied module
-            super(mod).tap do
-              # Take a snapshot of the list of singleton class ancestors
-              # after the actual include
-              after = singleton_class.ancestors
-              # The difference is the modules that are added to the list
-              # of ancestors of the singleton class. Those are all the
-              # modules that were `extend`ed due to the `include` call.
-              #
-              # We record those modules on our lookup table keyed by
-              # the included module with the values being all the modules
-              # that that module pulls into the singleton class.
-              #
-              # We need to reverse the order, since the extend order should
-              # be the inverse of the ancestor order. That is, earlier
-              # extended modules would be later in the ancestor chain.
-              mixins_from_modules[mod] = (after - before).reverse!
+            ::Tapioca::Runtime::Trackers::Mixin.with_disabled_registration do
+              super(mod).tap do
+                # Take a snapshot of the list of singleton class ancestors
+                # after the actual include
+                after = singleton_class.ancestors
+                # The difference is the modules that are added to the list
+                # of ancestors of the singleton class. Those are all the
+                # modules that were `extend`ed due to the `include` call.
+                #
+                # We record those modules on our lookup table keyed by
+                # the included module with the values being all the modules
+                # that that module pulls into the singleton class.
+                #
+                # We need to reverse the order, since the extend order should
+                # be the inverse of the ancestor order. That is, earlier
+                # extended modules would be later in the ancestor chain.
+                mixins_from_modules[mod] = (after - before).reverse!
+              end
             end
           rescue Exception # rubocop:disable Lint/RescueException
             # this is a best effort, bail if we can't perform this

data/lib/tapioca/runtime/reflection.rb

@@ -20,6 +20,8 @@ module Tapioca
       PRIVATE_INSTANCE_METHODS_METHOD = T.let(Module.instance_method(:private_instance_methods), UnboundMethod)
       METHOD_METHOD = T.let(Kernel.instance_method(:method), UnboundMethod)
 
+      REQUIRED_FROM_LABELS = T.let(["<top (required)>", "<main>"].freeze, T::Array[String])
+
       sig do
         params(
           symbol: String,
@@ -152,6 +154,30 @@ module Tapioca
 
         T.unsafe(result)
       end
+
+      # Examines the call stack to identify the closest location where a "require" is performed
+      # by searching for the label "<top (required)>". If none is found, it returns the location
+      # labeled "<main>", which is the original call site.
+      sig { params(locations: T.nilable(T::Array[Thread::Backtrace::Location])).returns(String) }
+      def resolve_loc(locations)
+        return "" unless locations
+
+        resolved_loc = locations.find { |loc| REQUIRED_FROM_LABELS.include?(loc.label) }
+        return "" unless resolved_loc
+
+        resolved_loc.absolute_path || ""
+      end
+
+      sig { params(constant: Module).returns(T.nilable(String)) }
+      def constant_name_from_singleton_class(constant)
+        constant.to_s.match("#<Class:(.+)>")&.captures&.first
+      end
+
+      sig { params(constant: Module).returns(T.nilable(BasicObject)) }
+      def constant_from_singleton_class(constant)
+        constant_name = constant_name_from_singleton_class(constant)
+        constantize(constant_name) if constant_name
+      end
     end
   end
 end

data/lib/tapioca/runtime/trackers/constant_definition.rb

@@ -1,8 +1,6 @@
 # typed: true
 # frozen_string_literal: true
 
-require "set"
-
 module Tapioca
   module Runtime
     module Trackers
@@ -12,31 +10,61 @@ module Tapioca
       # available in the ruby runtime without extra accounting.
       module ConstantDefinition
         extend Reflection
+        extend T::Sig
+
+        class ConstantLocation < T::Struct
+          const :lineno, Integer
+          const :path, String
+        end
 
-        @class_files = {}
+        @class_files = {}.compare_by_identity
 
         # Immediately activated upon load. Observes class/module definition.
         TracePoint.trace(:class) do |tp|
-          unless tp.self.singleton_class?
-            key = name_of(tp.self)
-            file = tp.path
-            if file == "(eval)"
-              file = T.must(caller_locations)
-                .drop_while { |loc| loc.path == "(eval)" }
-                .first&.path
-            end
-            @class_files[key] ||= Set.new
-            @class_files[key] << file
+          next if tp.self.singleton_class?
+
+          key = tp.self
+
+          path = tp.path
+          if File.exist?(path)
+            loc = build_constant_location(tp, caller_locations)
+          else
+            caller_location = T.must(caller_locations)
+              .find { |loc| loc.path && File.exist?(loc.path) }
+
+            next unless caller_location
+
+            loc = ConstantLocation.new(path: caller_location.absolute_path || "", lineno: caller_location.lineno)
           end
+
+          (@class_files[key] ||= Set.new) << loc
+        end
+
+        TracePoint.trace(:c_return) do |tp|
+          next unless tp.method_id == :new
+          next unless Module === tp.return_value
+
+          key = tp.return_value
+          loc = build_constant_location(tp, caller_locations)
+          (@class_files[key] ||= Set.new) << loc
+        end
+
+        def self.build_constant_location(tp, locations)
+          file = resolve_loc(caller_locations)
+          lineno = file == File.realpath(tp.path) ? tp.lineno : 0
+
+          ConstantLocation.new(path: file, lineno: lineno)
         end
 
         # Returns the files in which this class or module was opened. Doesn't know
         # about situations where the class was opened prior to +require+ing,
         # or where metaprogramming was used via +eval+, etc.
         def self.files_for(klass)
-          name = String === klass ? klass : name_of(klass)
-          files = @class_files[name]
-          files || Set.new
+          locations_for(klass).map(&:path).to_set
+        end
+
+        def self.locations_for(klass)
+          @class_files.fetch(klass, Set.new)
         end
       end
     end

data/lib/tapioca/runtime/trackers/mixin.rb

@@ -7,7 +7,9 @@ module Tapioca
       module Mixin
         extend T::Sig
 
-        @mixin_map = {}.compare_by_identity
+        @constants_to_mixin_locations = {}.compare_by_identity
+        @mixins_to_constants = {}.compare_by_identity
+        @enabled = true
 
         class Type < T::Enum
           enums do
@@ -17,24 +19,38 @@ module Tapioca
           end
         end
 
+        sig do
+          type_parameters(:Result)
+            .params(block: T.proc.returns(T.type_parameter(:Result)))
+            .returns(T.type_parameter(:Result))
+        end
+        def self.with_disabled_registration(&block)
+          @enabled = false
+
+          block.call
+        ensure
+          @enabled = true
+        end
+
         sig do
           params(
             constant: Module,
-            mod: Module,
+            mixin: Module,
             mixin_type: Type,
-            locations: T.nilable(T::Array[Thread::Backtrace::Location])
           ).void
         end
-        def self.register(constant, mod, mixin_type, locations)
-          locations ||= []
-          locations.map!(&:absolute_path).uniq!
-          locs = mixin_locations_for(constant)
-          locs.fetch(mixin_type).store(mod, T.cast(locations, T::Array[String]))
+        def self.register(constant, mixin, mixin_type)
+          return unless @enabled
+
+          location = Reflection.resolve_loc(caller_locations)
+
+          constants = constants_with_mixin(mixin)
+          constants.fetch(mixin_type).store(constant, location)
         end
 
-        sig { params(constant: Module).returns(T::Hash[Type, T::Hash[Module, T::Array[String]]]) }
-        def self.mixin_locations_for(constant)
-          @mixin_map[constant] ||= {
+        sig { params(mixin: Module).returns(T::Hash[Type, T::Hash[Module, String]]) }
+        def self.constants_with_mixin(mixin)
+          @mixins_to_constants[mixin] ||= {
             Type::Prepend => {}.compare_by_identity,
             Type::Include => {}.compare_by_identity,
             Type::Extend => {}.compare_by_identity,
@@ -52,8 +68,10 @@ class Module
         constant,
         self,
         Tapioca::Runtime::Trackers::Mixin::Type::Prepend,
-        caller_locations
       )
+
+      register_extend_on_attached_class(constant) if constant.singleton_class?
+
       super
     end
 
@@ -62,8 +80,10 @@ class Module
         constant,
         self,
         Tapioca::Runtime::Trackers::Mixin::Type::Include,
-        caller_locations
       )
+
+      register_extend_on_attached_class(constant) if constant.singleton_class?
+
       super
     end
 
@@ -72,9 +92,24 @@ class Module
         obj,
         self,
         Tapioca::Runtime::Trackers::Mixin::Type::Extend,
-        caller_locations
       ) if Module === obj
       super
     end
+
+    private
+
+    # Including or prepending on a singleton class is functionally equivalent to extending the
+    # attached class. Registering the mixin as an extend on the attached class ensures that
+    # this mixin can be found whether searching for an include/prepend on the singleton class
+    # or an extend on the attached class.
+    def register_extend_on_attached_class(constant)
+      attached_class = Tapioca::Runtime::Reflection.constant_from_singleton_class(constant)
+
+      Tapioca::Runtime::Trackers::Mixin.register(
+        T.cast(attached_class, Module),
+        self,
+        Tapioca::Runtime::Trackers::Mixin::Type::Extend,
+      ) if attached_class
+    end
   end)
 end

data/lib/tapioca/sorbet_ext/generic_name_patch.rb

@@ -1,9 +1,6 @@
 # typed: true
 # frozen_string_literal: true
 
-require "tapioca/sorbet_ext/name_patch"
-require "tapioca/helpers/sorbet_helper"
-
 module T
   module Generic
     # This module intercepts calls to generic type instantiations and type variable definitions.
@@ -174,7 +171,7 @@ module Tapioca
       lower = bounds[:lower].to_s if bounds.key?(:lower)
       upper = bounds[:upper].to_s if bounds.key?(:upper)
 
-      TypeVariableHelper.serialize_type_variable(
+      RBIHelper.serialize_type_variable(
         @type.serialize,
         @variance,
         fixed,