tapioca 0.7.3 → 0.8.0

data/lib/tapioca/cli.rb

@@ -5,6 +5,7 @@ module Tapioca
   class Cli < Thor
     include CliHelper
     include ConfigHelper
+    include SorbetHelper
     include ShimsHelper
 
     FILE_HEADER_OPTION_DESC = "Add a \"This file is generated\" header on top of each generated RBI file"
@@ -89,7 +90,7 @@ module Tapioca
     option :quiet,
       aliases: ["-q"],
       type: :boolean,
-      desc: "Supresses file creation output",
+      desc: "Suppresses file creation output",
       default: false
     option :workers,
       aliases: ["-w"],
@@ -171,7 +172,7 @@ module Tapioca
     option :doc,
       type: :boolean,
       desc: "Include YARD documentation from sources when generating RBIs. Warning: this might be slow",
-      default: false
+      default: true
     option :exported_gem_rbis,
       type: :boolean,
       desc: "Include RBIs found in the `rbi/` directory of the gem",
@@ -242,6 +243,7 @@ module Tapioca
     option :gem_rbi_dir, type: :string, desc: "Path to gem RBIs", default: DEFAULT_GEM_DIR
     option :dsl_rbi_dir, type: :string, desc: "Path to DSL RBIs", default: DEFAULT_DSL_DIR
     option :shim_rbi_dir, type: :string, desc: "Path to shim RBIs", default: DEFAULT_SHIM_DIR
+    option :payload, type: :boolean, desc: "Check shims against Sorbet's payload", default: true
     def check_shims
       index = RBI::Index.new
 
@@ -251,6 +253,30 @@ module Tapioca
         exit(0)
       end
 
+      payload_path = T.let(nil, T.nilable(String))
+
+      if options[:payload]
+        if sorbet_supports?(:print_payload_sources)
+          Dir.mktmpdir do |dir|
+            payload_path = dir
+            result = sorbet("--no-config --print=payload-sources:#{payload_path}")
+
+            unless result.status
+              say_error("Sorbet failed to dump payload")
+              say_error(result.err)
+              exit(1)
+            end
+
+            index_payload(index, payload_path)
+          end
+        else
+          say_error("The version of Sorbet used in your Gemfile.lock does not support `--print=payload-sources`")
+          say_error("Current: v#{SORBET_GEM_SPEC.version}")
+          say_error("Required: #{FEATURE_REQUIREMENTS[:print_payload_sources]}")
+          exit(1)
+        end
+      end
+
       index_rbis(index, "shim", shim_rbi_dir)
       index_rbis(index, "gem", options[:gem_rbi_dir])
       index_rbis(index, "dsl", options[:dsl_rbi_dir])
@@ -260,7 +286,11 @@ module Tapioca
         duplicates.each do |key, nodes|
           say_error("\nDuplicated RBI for #{key}:", :red)
           nodes.each do |node|
-            say_error(" * #{node.loc}", :red)
+            node_loc = node.loc
+            next unless node_loc
+
+            loc_string = location_to_payload_url(node_loc, path_prefix: payload_path)
+            say_error(" * #{loc_string}", :red)
           end
         end
         say_error("\nPlease remove the duplicated definitions from the #{shim_rbi_dir} directory.", :red)
@@ -271,6 +301,13 @@ module Tapioca
       exit(0)
     end
 
+    desc "annotations", "Pull gem annotations from a central RBI repository"
+    option :repo_uri, type: :string, desc: "Repository URI to pull annotations from", default: CENTRAL_REPO_ROOT_URI
+    def annotations
+      command = Commands::Annotations.new(central_repo_root_uri: options[:repo_uri])
+      command.execute
+    end
+
     map ["--version", "-v"] => :__print_version
 
     desc "--version, -v", "show version"

data/lib/tapioca/commands/annotations.rb

@@ -0,0 +1,154 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "net/http"
+
+module Tapioca
+  module Commands
+    class Annotations < Command
+      extend T::Sig
+
+      sig do
+        params(
+          central_repo_root_uri: String,
+          central_repo_index_path: String
+        ).void
+      end
+      def initialize(central_repo_root_uri:, central_repo_index_path: CENTRAL_REPO_INDEX_PATH)
+        super()
+        @central_repo_root_uri = central_repo_root_uri
+        @index = T.let(fetch_index, RepoIndex)
+      end
+
+      sig { override.void }
+      def execute
+        project_gems = list_gemfile_gems
+        remove_expired_annotations(project_gems)
+        fetch_annotations(project_gems)
+      end
+
+      private
+
+      sig { returns(T::Array[String]) }
+      def list_gemfile_gems
+        say("Listing gems from Gemfile.lock... ", [:blue, :bold])
+        gemfile = Bundler.read_file("Gemfile.lock")
+        parser = Bundler::LockfileParser.new(gemfile)
+        gem_names = parser.specs.map(&:name)
+        say("Done", :green)
+        gem_names
+      end
+
+      sig { params(project_gems: T::Array[String]).void }
+      def remove_expired_annotations(project_gems)
+        say("Removing annotations for gems that have been removed... ", [:blue, :bold])
+
+        annotations = Pathname.glob("#{DEFAULT_ANNOTATIONS_DIR}/*.rbi").map { |f| f.basename(".*").to_s }
+        expired = annotations - project_gems
+
+        if expired.empty?
+          say(" Nothing to do")
+          return
+        end
+
+        say("\n")
+        expired.each do |gem_name|
+          say("\n")
+          path = "#{DEFAULT_ANNOTATIONS_DIR}/#{gem_name}.rbi"
+          remove_file(path)
+        end
+        say("\nDone\n\n", :green)
+      end
+
+      sig { returns(RepoIndex) }
+      def fetch_index
+        say("Retrieving index from central repository... ", [:blue, :bold])
+        content = fetch_file(CENTRAL_REPO_INDEX_PATH)
+        exit(1) unless content
+
+        index = RepoIndex.from_json(content)
+        say("Done", :green)
+        index
+      end
+
+      sig { params(gem_names: T::Array[String]).returns(T::Array[String]) }
+      def fetch_annotations(gem_names)
+        say("Fetching gem annotations from central repository... ", [:blue, :bold])
+        fetchable_gems = gem_names.select { |gem_name| @index.has_gem?(gem_name) }
+
+        if fetchable_gems.empty?
+          say(" Nothing to do")
+          exit(0)
+        end
+
+        say("\n")
+        fetched_gems = fetchable_gems.select { |name| fetch_annotation(name) }
+        say("\nDone", :green)
+        fetched_gems
+      end
+
+      sig { params(gem_name: String).void }
+      def fetch_annotation(gem_name)
+        content = fetch_file("#{CENTRAL_REPO_ANNOTATIONS_DIR}/#{gem_name}.rbi")
+        return unless content
+
+        content = add_header(gem_name, content)
+
+        dir = DEFAULT_ANNOTATIONS_DIR
+        FileUtils.mkdir_p(dir)
+        say("\n  Fetched #{set_color(gem_name, :yellow, :bold)}", :green)
+        create_file("#{dir}/#{gem_name}.rbi", content)
+      end
+
+      sig { params(path: String).returns(T.nilable(String)) }
+      def fetch_file(path)
+        if @central_repo_root_uri.start_with?(%r{https?://})
+          fetch_http_file(path)
+        else
+          fetch_local_file(path)
+        end
+      end
+
+      sig { params(path: String).returns(T.nilable(String)) }
+      def fetch_local_file(path)
+        File.read("#{@central_repo_root_uri}/#{path}")
+      rescue => e
+        say_error("\nCan't fetch file `#{path}` (#{e.message})", :bold, :red)
+        nil
+      end
+
+      sig { params(path: String).returns(T.nilable(String)) }
+      def fetch_http_file(path)
+        uri = URI("#{@central_repo_root_uri}/#{path}")
+        response = Net::HTTP.get_response(uri)
+        case response
+        when Net::HTTPSuccess
+          response.body
+        else
+          say_error("\nCan't fetch file `#{path}` from #{@central_repo_root_uri} (#{response.class})", :bold, :red)
+          nil
+        end
+      rescue SocketError, Errno::ECONNREFUSED => e
+        say_error("\nCan't fetch file `#{path}` from #{@central_repo_root_uri} (#{e.message})", :bold, :red)
+        nil
+      end
+
+      sig { params(name: String, content: String).returns(String) }
+      def add_header(name, content)
+        header = <<~COMMENT
+          # DO NOT EDIT MANUALLY
+          # This file was pulled from #{@central_repo_root_uri}.
+          # Please run `#{default_command(:annotations)}` to update it.
+        COMMENT
+
+        contents = content.split("\n")
+        if contents[0]&.start_with?("# typed:") && contents[1]&.empty?
+          contents.insert(2, header).join("\n")
+        else
+          say_error("Couldn't insert file header for content: #{content} due to unexpected file format")
+          content
+        end
+      end
+    end
+  end
+end

data/lib/tapioca/commands/dsl.rb

@@ -4,6 +4,9 @@
 module Tapioca
   module Commands
     class Dsl < Command
+      include SorbetHelper
+      include RBIHelper
+
       sig do
         params(
           requested_constants: T::Array[String],
@@ -17,6 +20,8 @@ module Tapioca
           quiet: T::Boolean,
           verbose: T::Boolean,
           number_of_workers: T.nilable(Integer),
+          auto_strictness: T::Boolean,
+          gem_dir: String,
           rbi_formatter: RBIFormatter
         ).void
       end
@@ -32,6 +37,8 @@ module Tapioca
         quiet: false,
         verbose: false,
         number_of_workers: nil,
+        auto_strictness: true,
+        gem_dir: DEFAULT_GEM_DIR,
         rbi_formatter: DEFAULT_RBI_FORMATTER
       )
         @requested_constants = requested_constants
@@ -45,6 +52,8 @@ module Tapioca
         @quiet = quiet
         @verbose = verbose
         @number_of_workers = number_of_workers
+        @auto_strictness = auto_strictness
+        @gem_dir = gem_dir
         @rbi_formatter = rbi_formatter
 
         super()
@@ -102,9 +111,19 @@ module Tapioca
           perform_dsl_verification(outpath)
         else
           purge_stale_dsl_rbi_files(rbi_files_to_purge)
-
           say("Done", :green)
 
+          if @auto_strictness
+            say("")
+            validate_rbi_files(
+              command: default_command(:dsl, @requested_constants.join(" ")),
+              gem_dir: @gem_dir,
+              dsl_dir: @outpath.to_s,
+              auto_strictness: @auto_strictness,
+              compilers: pipeline.compilers
+            )
+          end
+
           say("All operations performed in working directory.", [:green, :bold])
           say("Please review changes and commit them.", [:green, :bold])
         end

data/lib/tapioca/commands/gem.rb

@@ -5,6 +5,7 @@ module Tapioca
   module Commands
     class Gem < Command
       include SorbetHelper
+      include RBIHelper
 
       sig do
         params(
@@ -78,7 +79,13 @@ module Tapioca
         end
 
         if anything_done
-          update_strictnesses(gem_queue.map(&:name), gem_dir: @outpath.to_s, dsl_dir: @dsl_dir) if @auto_strictness
+          validate_rbi_files(
+            command: default_command(:gem, @gem_names.join(" ")),
+            gem_dir: @outpath.to_s,
+            dsl_dir: @dsl_dir,
+            auto_strictness: @auto_strictness,
+            gems: bundle.dependencies
+          )
 
           say("All operations performed in working directory.", [:green, :bold])
           say("Please review changes and commit them.", [:green, :bold])
@@ -102,7 +109,13 @@ module Tapioca
         ].any?
 
         if anything_done
-          update_strictnesses([], gem_dir: @outpath.to_s, dsl_dir: @dsl_dir) if @auto_strictness
+          validate_rbi_files(
+            command: default_command(:gem),
+            gem_dir: @outpath.to_s,
+            dsl_dir: @dsl_dir,
+            auto_strictness: @auto_strictness,
+            gems: bundle.dependencies
+          )
 
           say("All operations performed in working directory.", [:green, :bold])
           say("Please review changes and commit them.", [:green, :bold])
@@ -122,7 +135,7 @@ module Tapioca
 
       sig { returns(Gemfile) }
       def bundle
-        @bundle ||= Gemfile.new
+        @bundle ||= Gemfile.new(@exclude)
       end
 
       sig { void }
@@ -359,6 +372,7 @@ module Tapioca
       sig { params(gem: Gemfile::GemSpec, file: RBI::File).void }
       def merge_with_exported_rbi(gem, file)
         return file unless gem.export_rbi_files?
+
         tree = gem.exported_rbi_tree
 
         unless tree.conflicts.empty?
@@ -379,60 +393,6 @@ module Tapioca
         say_error("\n\n  RBIs exported by `#{gem.name}` contain errors and can't be used:", :yellow)
         say_error("Cause: #{e.message} (#{e.location})")
       end
-
-      sig { params(gem_names: T::Array[String], gem_dir: String, dsl_dir: String).void }
-      def update_strictnesses(gem_names, gem_dir: DEFAULT_GEM_DIR, dsl_dir: DEFAULT_DSL_DIR)
-        return unless File.directory?(dsl_dir)
-
-        error_url_base = Spoom::Sorbet::Errors::DEFAULT_ERROR_URL_BASE
-
-        say("Typechecking RBI files... ")
-        res = sorbet(
-          "--no-config",
-          "--error-url-base=#{error_url_base}",
-          "--isolate-error-code 4010",
-          dsl_dir,
-          gem_dir
-        )
-        say(" Done", :green)
-
-        errors = Spoom::Sorbet::Errors::Parser.parse_string(res.err)
-
-        if errors.empty?
-          say("No error found", [:green, :bold])
-          return
-        end
-
-        files = []
-
-        errors.each do |error|
-          # Collect the file with error
-          files << error.file
-          error.more.each do |line|
-            # Also collect the conflicting definition file paths
-            next unless line.include?("Previous definition")
-            files << line.split(":").first&.strip
-          end
-        end
-
-        files
-          .uniq
-          .sort
-          .select do |file|
-            name = gem_name_from_rbi_path(file)
-            file.start_with?(gem_dir) && (gem_names.empty? || gem_names.include?(name))
-          end.each do |file|
-            Spoom::Sorbet::Sigils.change_sigil_in_file(file, "false")
-            say("\n  Changed strictness of #{file} to `typed: false` (conflicting with DSL files)", [:yellow, :bold])
-          end
-
-        say("\n")
-      end
-
-      sig { params(path: String).returns(String) }
-      def gem_name_from_rbi_path(path)
-        T.must(File.basename(path, ".rbi").split("@").first)
-      end
     end
   end
 end

data/lib/tapioca/commands/init.rb

@@ -41,6 +41,7 @@ module Tapioca
         create_file(@sorbet_config, <<~CONTENT, skip: true, force: false)
           --dir
           .
+          --ignore=vendor/
         CONTENT
       end
 

data/lib/tapioca/commands.rb

@@ -4,6 +4,7 @@
 module Tapioca
   module Commands
     autoload :Command, "tapioca/commands/command"
+    autoload :Annotations, "tapioca/commands/annotations"
     autoload :Dsl, "tapioca/commands/dsl"
     autoload :Init, "tapioca/commands/init"
     autoload :Gem, "tapioca/commands/gem"

data/lib/tapioca/dsl/compilers/protobuf.rb

@@ -83,6 +83,7 @@ module Tapioca
               create_type_members(klass, "Key", "Value")
             else
               descriptor = T.let(T.unsafe(constant).descriptor, Google::Protobuf::Descriptor)
+              descriptor.each_oneof { |oneof| create_oneof_method(klass, oneof) }
               fields = descriptor.map { |desc| create_descriptor_method(klass, desc) }
               fields.sort_by!(&:name)
 
@@ -216,6 +217,19 @@ module Tapioca
 
           field
         end
+
+        sig do
+          params(
+            klass: RBI::Scope,
+            desc: Google::Protobuf::OneofDescriptor
+          ).void
+        end
+        def create_oneof_method(klass, desc)
+          klass.create_method(
+            desc.name,
+            return_type: "T.nilable(Symbol)"
+          )
+        end
       end
     end
   end

data/lib/tapioca/dsl/pipeline.rb

@@ -149,8 +149,12 @@ module Tapioca
 
         compilers.each do |compiler_class|
           next unless compiler_class.handles?(constant)
+
           compiler = compiler_class.new(self, file.root, constant)
           compiler.decorate
+        rescue
+          $stderr.puts("Error: `#{compiler_class.name}` failed to generate RBI for `#{constant}`")
+          raise # This is an unexpected error, so re-raise it
         end
 
         return if file.root.empty?

data/lib/tapioca/dsl.rb

@@ -0,0 +1,6 @@
+# typed: true
+# frozen_string_literal: true
+
+require "tapioca"
+require "tapioca/runtime/reflection"
+require "tapioca/dsl/compiler"

data/lib/tapioca/executor.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "etc"
+require "parallel"
 
 module Tapioca
   class Executor
@@ -20,10 +21,6 @@ module Tapioca
         number_of_workers || [Etc.nprocessors, (queue.length.to_f / MINIMUM_ITEMS_PER_WORKER).ceil].min,
         Integer
       )
-
-      # The number of items that will be processed per worker, so that we can split the queue into groups and assign
-      # them to each one of the workers
-      @items_per_worker = T.let((queue.length.to_f / @number_of_workers).ceil, Integer)
     end
 
     sig do
@@ -32,48 +29,9 @@ module Tapioca
       ).returns(T::Array[T.type_parameter(:T)])
     end
     def run_in_parallel(&block)
-      # If we only have one worker selected, it's not worth forking, just run sequentially
-      return @queue.map { |item| block.call(item) } if @number_of_workers == 1
-
-      read_pipes = []
-      write_pipes = []
-
-      # If we have more than one worker, fork the pool by shifting the expected number of items per worker from the
-      # queue
-      workers = (0...@number_of_workers).map do
-        items = @queue.shift(@items_per_worker)
-
-        # Each worker has their own pair of pipes, so that we can read the result from each worker separately
-        read, write = IO.pipe
-        read_pipes << read
-        write_pipes << write
-
-        fork do
-          read.close
-          result = items.map { |item| block.call(item) }
-
-          # Pack the result as a Base64 string of the Marshal dump of the array of values returned by the block that we
-          # ran in parallel
-          packed = [Marshal.dump(result)].pack("m")
-          write.puts(packed)
-          write.close
-        end
-      end
-
-      # Close all the write pipes, then read and close from all the read pipes
-      write_pipes.each(&:close)
-      result = read_pipes.map do |pipe|
-        content = pipe.read
-        pipe.close
-        content
-      end
-
-      # Wait until all the workers finish. Notice that waiting for the PIDs can only happen after we read and close the
-      # pipe or else we may end up in a condition where writing to the pipe hangs indefinitely
-      workers.each { |pid| Process.waitpid(pid) }
-
-      # Decode the value back into the Ruby objects by doing the inverse of what each worker does
-      result.flat_map { |item| T.unsafe(Marshal.load(item.unpack1("m"))) }
+      # To have the parallel gem run jobs in the parent process, you must pass 0 as the number of processes
+      number_of_processes = @number_of_workers == 1 ? 0 : @number_of_workers
+      Parallel.map(@queue, { in_processes: number_of_processes }, &block)
     end
   end
 end

data/lib/tapioca/gem/listeners/methods.rb

@@ -41,6 +41,7 @@ module Tapioca
             .each do |visibility, method_list|
               method_list.sort!.map do |name|
                 next if name == :initialize
+
                 vis = case visibility
                 when :protected
                   RBI::Protected.new
@@ -65,7 +66,7 @@ module Tapioca
         end
         def compile_method(tree, symbol_name, constant, method, visibility = RBI::Public.new)
           return unless method
-          return unless method.owner == constant
+          return unless method_owned_by_constant?(method, constant)
           return if @pipeline.symbol_in_payload?(symbol_name) && !@pipeline.method_in_gem?(method)
 
           signature = signature_of(method)
@@ -141,6 +142,29 @@ module Tapioca
           tree << rbi_method
         end
 
+        # Check whether the method is defined by the constant.
+        #
+        # In most cases, it works to check that the constant is the method owner. However,
+        # in the case that a method is also defined in a module prepended to the constant, it
+        # will be owned by the prepended module, not the constant.
+        #
+        # This method implements a better way of checking whether a constant defines a method.
+        # It walks up the ancestor tree via the `super_method` method; if any of the super
+        # methods are owned by the constant, it means that the constant declares the method.
+        sig { params(method: UnboundMethod, constant: Module).returns(T::Boolean) }
+        def method_owned_by_constant?(method, constant)
+          # Widen the type of `method` to be nilable
+          method = T.let(method, T.nilable(UnboundMethod))
+
+          while method
+            return true if method.owner == constant
+
+            method = method.super_method
+          end
+
+          false
+        end
+
         sig { params(mod: Module).returns(T::Hash[Symbol, T::Array[Symbol]]) }
         def method_names_by_visibility(mod)
           {
@@ -163,6 +187,7 @@ module Tapioca
         sig { params(name: String).returns(T::Boolean) }
         def valid_method_name?(name)
           return true if SPECIAL_METHOD_NAMES.include?(name)
+
           !!name.match(/^[[:word:]]+[?!=]?$/)
         end
 

data/lib/tapioca/gem/listeners/sorbet_props.rb

@@ -19,7 +19,7 @@ module Tapioca
           constant.props.map do |name, prop|
             type = prop.fetch(:type_object, "T.untyped").to_s.gsub(".returns(<VOID>)", ".void")
 
-            default = prop.key?(:default) ? "T.unsafe(nil)" : nil
+            default = prop.key?(:default) || prop.key?(:factory) ? "T.unsafe(nil)" : nil
             node << if prop.fetch(:immutable, false)
               RBI::TStructConst.new(name.to_s, type, default: default)
             else

data/lib/tapioca/gem/listeners/sorbet_required_ancestors.rb

@@ -14,6 +14,7 @@ module Tapioca
           ancestors = Runtime::Trackers::RequiredAncestor.required_ancestors_by(event.constant)
           ancestors.each do |ancestor|
             next unless ancestor # TODO: We should have a way to warn from here
+
             event.node << RBI::RequiresAncestor.new(ancestor.to_s)
           end
         end

data/lib/tapioca/gem/pipeline.rb

@@ -87,6 +87,7 @@ module Tapioca
       def symbol_in_payload?(symbol_name)
         symbol_name = symbol_name[2..-1] if symbol_name.start_with?("::")
         return false unless symbol_name
+
         @payload_symbols.include?(symbol_name)
       end
 
@@ -102,9 +103,11 @@ module Tapioca
       def name_of(constant)
         name = name_of_proxy_target(constant, super(class_of(constant)))
         return name if name
+
         name = super(constant)
         return if name.nil?
         return unless are_equal?(constant, constantize(name, inherit: true))
+
         name = "Struct" if name =~ /^(::)?Struct::[^:]+$/
         name
       end
@@ -350,6 +353,7 @@ module Tapioca
       sig { params(constant: Module, class_name: T.nilable(String)).returns(T.nilable(String)) }
       def name_of_proxy_target(constant, class_name)
         return unless class_name == "ActiveSupport::Deprecation::DeprecatedConstantProxy"
+
         # We are dealing with a ActiveSupport::Deprecation::DeprecatedConstantProxy
         # so try to get the name of the target class
         begin