tapioca 0.5.6 → 0.6.0

data/lib/tapioca/compilers/symbol_table/symbol_generator.rb

@@ -85,26 +85,13 @@ module Tapioca
 
         sig { params(tree: RBI::Tree, symbol: String).void }
         def generate_from_symbol(tree, symbol)
-          constant = resolve_constant(symbol)
+          constant = constantize(symbol)
 
           return unless constant
 
           compile(tree, symbol, constant)
         end
 
-        sig do
-          params(
-            symbol: String,
-            inherit: T::Boolean,
-            namespace: Module
-          ).returns(BasicObject).checked(:never)
-        end
-        def resolve_constant(symbol, inherit: false, namespace: Object)
-          namespace.const_get(symbol, inherit)
-        rescue NameError, LoadError, RuntimeError, ArgumentError, TypeError
-          nil
-        end
-
         sig { params(tree: RBI::Tree, name: T.nilable(String), constant: BasicObject).void.checked(:never) }
         def compile(tree, name, constant)
           return unless constant
@@ -269,7 +256,7 @@ module Tapioca
         def compile_subconstants(tree, name, constant)
           constants_of(constant).sort.uniq.map do |constant_name|
             symbol = (name == "Object" ? "" : name) + "::#{constant_name}"
-            subconstant = resolve_constant(symbol)
+            subconstant = constantize(symbol)
 
             # Don't compile modules of Object because Object::Foo == Foo
             # Don't compile modules of BasicObject because BasicObject::BasicObject == BasicObject
@@ -300,7 +287,7 @@ module Tapioca
           # variable via the value of the type variable constant.
           subconstant_to_name_lookup = constants_of(constant)
             .each_with_object({}.compare_by_identity) do |constant_name, table|
-            table[resolve_constant(constant_name.to_s, namespace: constant)] = constant_name.to_s
+            table[constantize(constant_name.to_s, namespace: constant)] = constant_name.to_s
           end
 
           # Map each type variable to its string representation.
@@ -356,7 +343,7 @@ module Tapioca
             superclass_name = name_of(superclass)
             next unless superclass_name
 
-            resolved_superclass = resolve_constant(superclass_name)
+            resolved_superclass = constantize(superclass_name)
             next unless Module === resolved_superclass
             next if name_of(resolved_superclass) == constant_name
 
@@ -382,43 +369,44 @@ module Tapioca
           interesting_ancestors = interesting_ancestors_of(constant)
           interesting_singleton_class_ancestors = interesting_ancestors_of(singleton_class)
 
-          prepend = interesting_ancestors.take_while { |c| !are_equal?(constant, c) }
-          include = interesting_ancestors.drop(prepend.size + 1)
-          extend  = interesting_singleton_class_ancestors.reject do |mod|
+          prepends = interesting_ancestors.take_while { |c| !are_equal?(constant, c) }
+          includes = interesting_ancestors.drop(prepends.size + 1)
+          extends  = interesting_singleton_class_ancestors.reject do |mod|
             Module != class_of(mod) || are_equal?(mod, singleton_class)
           end
 
-          prepend
-            .reverse
-            .select { |mod| (name = name_of(mod)) && !name.start_with?("T::") }
-            .map do |mod|
-              add_to_symbol_queue(name_of(mod))
-
-              # TODO: Sorbet currently does not handle prepend
-              # properly for method resolution, so we generate an
-              # include statement instead
-              qname = qualified_name_of(mod)
-              tree << RBI::Include.new(T.must(qname))
-            end
+          add_mixins(tree, prepends.reverse, Trackers::Mixin::Type::Prepend)
+          add_mixins(tree, includes.reverse, Trackers::Mixin::Type::Include)
+          add_mixins(tree, extends.reverse, Trackers::Mixin::Type::Extend)
+        end
 
-          include
-            .reverse
-            .select { |mod| (name = name_of(mod)) && !name.start_with?("T::") }
-            .map do |mod|
-              add_to_symbol_queue(name_of(mod))
+        sig do
+          params(
+            tree: RBI::Tree,
+            mods: T::Array[Module],
+            mixin_type: Trackers::Mixin::Type
+          ).void
+        end
+        def add_mixins(tree, mods, mixin_type)
+          mods
+            .select do |mod|
+              name = name_of(mod)
 
-              qname = qualified_name_of(mod)
-              tree << RBI::Include.new(T.must(qname))
+              name && !name.start_with?("T::")
             end
-
-          extend
-            .reverse
-            .select { |mod| (name = name_of(mod)) && !name.start_with?("T::") }
             .map do |mod|
               add_to_symbol_queue(name_of(mod))
 
               qname = qualified_name_of(mod)
-              tree << RBI::Extend.new(T.must(qname))
+              case mixin_type
+              # TODO: Sorbet currently does not handle prepend
+              # properly for method resolution, so we generate an
+              # include statement instead
+              when Trackers::Mixin::Type::Include, Trackers::Mixin::Type::Prepend
+                tree << RBI::Include.new(T.must(qname))
+              when Trackers::Mixin::Type::Extend
+                tree << RBI::Extend.new(T.must(qname))
+              end
             end
         end
 
@@ -647,7 +635,7 @@ module Tapioca
         sig { params(constant: Module, strict: T::Boolean).returns(T::Boolean) }
         def defined_in_gem?(constant, strict: true)
           files = Set.new(get_file_candidates(constant))
-            .merge(Tapioca::ConstantLocator.files_for(constant))
+            .merge(Tapioca::Trackers::ConstantDefinition.files_for(constant))
 
           return !strict if files.empty?
 
@@ -656,6 +644,19 @@ module Tapioca
           end
         end
 
+        sig do
+          params(
+            mod: Module,
+            mixin_type: Trackers::Mixin::Type,
+            mixin_locations: T::Hash[Trackers::Mixin::Type, T::Hash[Module, T::Array[String]]]
+          ).returns(T::Boolean)
+        end
+        def mixed_in_by_gem?(mod, mixin_type, mixin_locations)
+          locations = mixin_locations.dig(mixin_type, mod)
+          return true unless locations
+          locations.any? { |location| gem.contains_path?(location) }
+        end
+
         sig { params(constant: Module).returns(T::Array[String]) }
         def get_file_candidates(constant)
           wrapped_module = Pry::WrappedModule.new(constant)
@@ -727,7 +728,7 @@ module Tapioca
           return name if name
           name = super(constant)
           return if name.nil?
-          return unless are_equal?(constant, resolve_constant(name, inherit: true))
+          return unless are_equal?(constant, constantize(name, inherit: true))
           name = "Struct" if name =~ /^(::)?Struct::[^:]+$/
           name
         end

data/lib/tapioca/executor.rb

@@ -0,0 +1,79 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "etc"
+
+module Tapioca
+  class Executor
+    extend T::Sig
+
+    MINIMUM_ITEMS_PER_WORKER = T.let(2, Integer)
+
+    sig { params(queue: T::Array[T.untyped], number_of_workers: T.nilable(Integer)).void }
+    def initialize(queue, number_of_workers: nil)
+      @queue = queue
+
+      # Forking workers is expensive and not worth it for a low number of gems. Here we assign the number of workers to
+      # be the minimum between the number of available processors (max) or the number of workers to make sure that each
+      # one has at least 4 items to process
+      @number_of_workers = T.let(
+        number_of_workers || [Etc.nprocessors, (queue.length.to_f / MINIMUM_ITEMS_PER_WORKER).ceil].min,
+        Integer
+      )
+
+      # The number of items that will be processed per worker, so that we can split the queue into groups and assign
+      # them to each one of the workers
+      @items_per_worker = T.let((queue.length.to_f / @number_of_workers).ceil, Integer)
+    end
+
+    sig do
+      type_parameters(:T).params(
+        block: T.proc.params(item: T.untyped).returns(T.type_parameter(:T))
+      ).returns(T::Array[T.type_parameter(:T)])
+    end
+    def run_in_parallel(&block)
+      # If we only have one worker selected, it's not worth forking, just run sequentially
+      return @queue.map { |item| block.call(item) } if @number_of_workers == 1
+
+      read_pipes = []
+      write_pipes = []
+
+      # If we have more than one worker, fork the pool by shifting the expected number of items per worker from the
+      # queue
+      workers = (0...@number_of_workers).map do
+        items = @queue.shift(@items_per_worker)
+
+        # Each worker has their own pair of pipes, so that we can read the result from each worker separately
+        read, write = IO.pipe
+        read_pipes << read
+        write_pipes << write
+
+        fork do
+          read.close
+          result = items.map { |item| block.call(item) }
+
+          # Pack the result as a Base64 string of the Marshal dump of the array of values returned by the block that we
+          # ran in parallel
+          packed = [Marshal.dump(result)].pack("m")
+          write.puts(packed)
+          write.close
+        end
+      end
+
+      # Close all the write pipes, then read and close from all the read pipes
+      write_pipes.each(&:close)
+      result = read_pipes.map do |pipe|
+        content = pipe.read
+        pipe.close
+        content
+      end
+
+      # Wait until all the workers finish. Notice that waiting for the PIDs can only happen after we read and close the
+      # pipe or else we may end up in a condition where writing to the pipe hangs indefinitely
+      workers.each { |pid| Process.waitpid(pid) }
+
+      # Decode the value back into the Ruby objects by doing the inverse of what each worker does
+      result.flat_map { |item| T.unsafe(Marshal.load(item.unpack1("m"))) }
+    end
+  end
+end

data/lib/tapioca/gemfile.rb

@@ -105,6 +105,7 @@ module Tapioca
         real_gem_path = to_realpath(@spec.full_gem_path)
         @full_gem_path = T.let(real_gem_path, String)
         @version = T.let(version_string, String)
+        @exported_rbi_files = T.let(nil, T.nilable(T::Array[String]))
       end
 
       sig { params(gemfile_dir: String).returns(T::Boolean) }
@@ -151,6 +152,28 @@ module Tapioca
         files.each { |path| YARD.parse(path.to_s, [], Logger::Severity::FATAL) }
       end
 
+      sig { returns(T::Array[String]) }
+      def exported_rbi_files
+        @exported_rbi_files ||= Dir.glob("#{full_gem_path}/rbi/**/*.rbi")
+      end
+
+      sig { returns(T::Boolean) }
+      def export_rbi_files?
+        exported_rbi_files.any?
+      end
+
+      sig { returns(RBI::MergeTree) }
+      def exported_rbi_tree
+        rewriter = RBI::Rewriters::Merge.new(keep: RBI::Rewriters::Merge::Keep::NONE)
+
+        exported_rbi_files.each do |file|
+          rbi = RBI::Parser.parse_file(file)
+          rewriter.merge(rbi)
+        end
+
+        rewriter.tree
+      end
+
       private
 
       sig { returns(T::Boolean) }

data/lib/tapioca/generators/base.rb

@@ -1,9 +1,6 @@
 # typed: strict
 # frozen_string_literal: true
 
-# TODO: Remove me when logging logic has been abstracted.
-require "thor"
-
 module Tapioca
   module Generators
     class Base
@@ -14,7 +11,7 @@ module Tapioca
         include Thor::Actions
       end
 
-      # TODO: Remove me when logging logic has been abstracted
+      include CliHelper
       include Thor::Base
 
       abstract!
@@ -30,20 +27,6 @@ module Tapioca
 
       private
 
-      # TODO: Remove me when logging logic has been abstracted
-      sig { params(message: String, color: T.any(Symbol, T::Array[Symbol])).void }
-      def say_error(message = "", *color)
-        force_new_line = (message.to_s !~ /( |\t)\Z/)
-        # NOTE: This is a hack. We're no longer subclassing from Thor::Shell::Color
-        # so we no longer have access to the prepare_message call.
-        # We should update this to remove this.
-        buffer = shell.send(:prepare_message, *T.unsafe([message, *T.unsafe(color)]))
-        buffer << "\n" if force_new_line && !message.to_s.end_with?("\n")
-
-        $stderr.print(buffer)
-        $stderr.flush
-      end
-
       sig do
         params(
           path: T.any(String, Pathname),
@@ -56,6 +39,16 @@ module Tapioca
       def create_file(path, content, force: true, skip: false, verbose: true)
         @file_writer.create_file(path, force: force, skip: skip, verbose: verbose) { content }
       end
+
+      sig do
+        params(
+          path: T.any(String, Pathname),
+          verbose: T::Boolean
+        ).void
+      end
+      def remove_file(path, verbose: true)
+        @file_writer.remove_file(path, verbose: verbose)
+      end
     end
   end
 end

data/lib/tapioca/generators/dsl.rb

@@ -8,8 +8,8 @@ module Tapioca
         params(
           requested_constants: T::Array[String],
           outpath: Pathname,
-          generators: T::Array[String],
-          exclude_generators: T::Array[String],
+          only: T::Array[String],
+          exclude: T::Array[String],
           file_header: T::Boolean,
           compiler_path: String,
           tapioca_path: String,
@@ -17,14 +17,15 @@ module Tapioca
           file_writer: Thor::Actions,
           should_verify: T::Boolean,
           quiet: T::Boolean,
-          verbose: T::Boolean
+          verbose: T::Boolean,
+          number_of_workers: T.nilable(Integer),
         ).void
       end
       def initialize(
         requested_constants:,
         outpath:,
-        generators:,
-        exclude_generators:,
+        only:,
+        exclude:,
         file_header:,
         compiler_path:,
         tapioca_path:,
@@ -32,18 +33,20 @@ module Tapioca
         file_writer: FileWriter.new,
         should_verify: false,
         quiet: false,
-        verbose: false
+        verbose: false,
+        number_of_workers: nil
       )
         @requested_constants = requested_constants
         @outpath = outpath
-        @generators = generators
-        @exclude_generators = exclude_generators
+        @only = only
+        @exclude = exclude
         @file_header = file_header
         @compiler_path = compiler_path
         @tapioca_path = tapioca_path
         @should_verify = should_verify
         @quiet = quiet
         @verbose = verbose
+        @number_of_workers = number_of_workers
 
         super(default_command: default_command, file_writer: file_writer)
 
@@ -52,6 +55,7 @@ module Tapioca
 
       sig { override.void }
       def generate
+        load_dsl_extensions
         load_application(eager_load: @requested_constants.empty?)
         abort_if_pending_migrations!
         load_dsl_generators
@@ -68,31 +72,31 @@ module Tapioca
 
         compiler = Compilers::DslCompiler.new(
           requested_constants: constantize(@requested_constants),
-          requested_generators: constantize_generators(@generators),
-          excluded_generators: constantize_generators(@exclude_generators),
+          requested_generators: constantize_generators(@only),
+          excluded_generators: constantize_generators(@exclude),
           error_handler: ->(error) {
             say_error(error, :bold, :red)
-          }
+          },
+          number_of_workers: @number_of_workers
         )
 
-        compiler.run do |constant, contents|
+        processed_files = compiler.run do |constant, contents|
           constant_name = T.must(Reflection.name_of(constant))
 
           if @verbose && !@quiet
             say_status(:processing, constant_name, :yellow)
           end
 
-          filename = compile_dsl_rbi(
+          compile_dsl_rbi(
             constant_name,
             contents,
             outpath: outpath,
             quiet: @should_verify || @quiet && !@verbose
           )
-
-          if filename
-            rbi_files_to_purge.delete(filename)
-          end
         end
+
+        processed_files.each { |filename| rbi_files_to_purge.delete(T.must(filename)) }
+
         say("")
 
         if @should_verify
@@ -171,7 +175,8 @@ module Tapioca
         unless unprocessable_constants.empty?
           unprocessable_constants.each do |name, _|
             say("Error: Cannot find constant '#{name}'", :red)
-            remove(dsl_rbi_filename(name))
+            filename = dsl_rbi_filename(name)
+            remove_file(filename) if File.file?(filename)
           end
 
           exit(1)
@@ -182,17 +187,9 @@ module Tapioca
 
       sig { params(generator_names: T::Array[String]).returns(T::Array[T.class_of(Compilers::Dsl::Base)]) }
       def constantize_generators(generator_names)
-        generator_map = generator_names.map do |name|
-          # Try to find built-in tapioca generator first, then globally defined generator. The
-          # explicit `break` ensures the class is returned, not the `potential_name`.
-          generator_klass = ["Tapioca::Compilers::Dsl::#{name}", name].find do |potential_name|
-            break Object.const_get(potential_name)
-          rescue NameError
-            # Skip if we can't find generator by the potential name
-          end
-
-          [name, generator_klass]
-        end.to_h
+        generator_map = generator_names.to_h do |name|
+          [name, Compilers::Dsl::Base.resolve(name)]
+        end
 
         unprocessable_generators = generator_map.select { |_, v| v.nil? }
         unless unprocessable_generators.empty?
@@ -203,7 +200,7 @@ module Tapioca
           exit(1)
         end
 
-        generator_map.values
+        T.cast(generator_map.values, T::Array[T.class_of(Compilers::Dsl::Base)])
       end
 
       sig do
@@ -245,7 +242,7 @@ module Tapioca
           say("Removing stale RBI files...")
 
           files.sort.each do |filename|
-            remove(filename)
+            remove_file(filename)
           end
           say("")
         end
@@ -256,13 +253,6 @@ module Tapioca
         @outpath / "#{underscore(constant_name)}.rbi"
       end
 
-      sig { params(filename: Pathname).void }
-      def remove(filename)
-        return unless filename.exist?
-        say("-- Removing: #{filename}")
-        filename.unlink
-      end
-
       sig { params(tmp_dir: Pathname).returns(T::Hash[String, Symbol]) }
       def verify_dsl_rbi(tmp_dir:)
         diff = {}
@@ -357,6 +347,11 @@ module Tapioca
       def generate_command_for(constant)
         "#{@default_command} dsl #{constant}"
       end
+
+      sig { void }
+      def load_dsl_extensions
+        Dir["#{__dir__}/../compilers/dsl/extensions/*.rb"].sort.each { |f| require(f) }
+      end
     end
   end
 end