tapioca 0.2.8 → 0.4.2

data/lib/tapioca/compilers/symbol_table/symbol_loader.rb

@@ -2,16 +2,12 @@
 # typed: true
 
 require 'json'
-require 'pathname'
 require 'tempfile'
-require 'shellwords'
 
 module Tapioca
   module Compilers
     module SymbolTable
       module SymbolLoader
-        SORBET = Pathname.new(Gem::Specification.find_by_name("sorbet-static").full_gem_path) / "libexec" / "sorbet"
-
         class << self
           extend(T::Sig)
 
@@ -33,7 +29,7 @@ module Tapioca
               file.write(Array(paths).join("\n"))
               file.flush
 
-              symbol_table_json_from("@#{file.path}")
+              symbol_table_json_from("@#{file.path.shellescape}")
             end, T.nilable(String))
 
             return Set.new if output.nil? || output.empty?
@@ -44,7 +40,7 @@ module Tapioca
 
           def ignored_symbols
             unless @ignored_symbols
-              output = symbol_table_json_from("''", table_type: "symbol-table-full-json")
+              output = symbol_table_json_from("-e ''", table_type: "symbol-table-full-json")
               json = JSON.parse(output)
               @ignored_symbols = SymbolTableParser.parse(json)
             end
@@ -53,17 +49,7 @@ module Tapioca
           end
 
           def symbol_table_json_from(input, table_type: "symbol-table-json")
-            IO.popen(
-              [
-                SORBET,
-                # We don't want to pick up any sorbet/config files in cwd
-                "--no-config",
-                "--print=#{table_type}",
-                "--quiet",
-                input,
-              ].shelljoin,
-              err: "/dev/null"
-            ).read
+            Tapioca::Compilers::Sorbet.run("--no-config", "--print=#{table_type}", input)
           end
         end
 

data/lib/tapioca/compilers/todos_compiler.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+# typed: strong
+
+module Tapioca
+  module Compilers
+    # Taken from https://github.com/sorbet/sorbet/blob/master/gems/sorbet/lib/todo-rbi.rb
+    class TodosCompiler
+      extend(T::Sig)
+
+      sig do
+        returns(String)
+      end
+      def compile
+        list_todos.each_line.map do |line|
+          next if line.include?("<") || line.include?("class_of")
+          "module #{line.strip.gsub('T.untyped::', '')}; end"
+        end.compact.join("\n")
+      end
+
+      private
+
+      sig { returns(String) }
+      def list_todos
+        Tapioca::Compilers::Sorbet.run(
+          "--print=missing-constants",
+          "--stdout-hup-hack",
+          "--no-error-count"
+        ).strip
+      end
+    end
+  end
+end

data/lib/tapioca/config.rb

@@ -0,0 +1,42 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Tapioca
+  class Config < T::Struct
+    extend(T::Sig)
+
+    const(:outdir, String)
+    const(:prerequire, T.nilable(String))
+    const(:postrequire, String)
+    const(:generate_command, String)
+    const(:exclude, T::Array[String])
+    const(:typed_overrides, T::Hash[String, String])
+    const(:todos_path, String)
+    const(:generators, T::Array[String])
+
+    sig { returns(Pathname) }
+    def outpath
+      @outpath = T.let(@outpath, T.nilable(Pathname))
+      @outpath ||= Pathname.new(outdir)
+    end
+
+    private_class_method :new
+
+    SORBET_PATH = T.let("sorbet", String)
+    SORBET_CONFIG = T.let("#{SORBET_PATH}/config", String)
+    TAPIOCA_PATH = T.let("#{SORBET_PATH}/tapioca", String)
+    TAPIOCA_CONFIG = T.let("#{TAPIOCA_PATH}/config.yml", String)
+
+    DEFAULT_POSTREQUIRE = T.let("#{TAPIOCA_PATH}/require.rb", String)
+    DEFAULT_RBIDIR = T.let("#{SORBET_PATH}/rbi", String)
+    DEFAULT_DSLDIR = T.let("#{DEFAULT_RBIDIR}/dsl", String)
+    DEFAULT_GEMDIR = T.let("#{DEFAULT_RBIDIR}/gems", String)
+    DEFAULT_TODOSPATH = T.let("#{DEFAULT_RBIDIR}/todo.rbi", String)
+
+    DEFAULT_OVERRIDES = T.let({
+      # ActiveSupport overrides some core methods with different signatures
+      # so we generate a typed: false RBI for it to suppress errors
+      "activesupport" => "false",
+    }.freeze, T::Hash[String, String])
+  end
+end

data/lib/tapioca/config_builder.rb

@@ -0,0 +1,75 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Tapioca
+  class ConfigBuilder
+    class << self
+      extend(T::Sig)
+
+      sig { params(command: Symbol, options: T::Hash[String, T.untyped]).returns(Config) }
+      def from_options(command, options)
+        Config.from_hash(
+          merge_options(default_options(command), config_options, options)
+        )
+      end
+
+      private
+
+      sig { returns(T::Hash[String, T.untyped]) }
+      def config_options
+        if File.exist?(Config::TAPIOCA_CONFIG)
+          YAML.load_file(Config::TAPIOCA_CONFIG, fallback: {})
+        else
+          {}
+        end
+      end
+
+      sig { params(command: Symbol).returns(T::Hash[String, T.untyped]) }
+      def default_options(command)
+        default_outdir = case command
+        when :sync, :generate
+          Config::DEFAULT_GEMDIR
+        when :dsl
+          Config::DEFAULT_DSLDIR
+        else
+          Config::SORBET_PATH
+        end
+
+        DEFAULT_OPTIONS.merge("outdir" => default_outdir)
+      end
+
+      sig { returns(String) }
+      def default_command
+        command = File.basename($PROGRAM_NAME)
+        args = ARGV.join(" ")
+
+        "#{command} #{args}".strip
+      end
+
+      sig { params(options: T::Hash[String, T.untyped]).returns(T::Hash[String, T.untyped]) }
+      def merge_options(*options)
+        options.each_with_object({}) do |option, result|
+          result.merge!(option) do |_, this_val, other_val|
+            if this_val.is_a?(Hash) && other_val.is_a?(Hash)
+              this_val.merge(other_val)
+            else
+              other_val
+            end
+          end
+        end
+      end
+    end
+
+    DEFAULT_OPTIONS = T.let({
+      "postrequire" => Config::DEFAULT_POSTREQUIRE,
+      "outdir" => nil,
+      "generate_command" => default_command,
+      "exclude" => [],
+      "typed_overrides" => Config::DEFAULT_OVERRIDES,
+      "todos_path" => Config::DEFAULT_TODOSPATH,
+      "generators" => [],
+    }.freeze, T::Hash[String, T.untyped])
+  end
+end

data/lib/tapioca/constant_locator.rb

@@ -1,3 +1,4 @@
+# typed: true
 # frozen_string_literal: true
 
 require 'set'

data/lib/tapioca/core_ext/class.rb

@@ -0,0 +1,23 @@
+# typed: false
+# frozen_string_literal: true
+
+class Class
+  # Returns an array with all classes that are < than its receiver.
+  #
+  #   class C; end
+  #   C.descendants # => []
+  #
+  #   class B < C; end
+  #   C.descendants # => [B]
+  #
+  #   class A < B; end
+  #   C.descendants # => [B, A]
+  #
+  #   class D < C; end
+  #   C.descendants # => [B, A, D]
+  def descendants
+    ObjectSpace.each_object(singleton_class).reject do |k|
+      k.singleton_class? || k == self
+    end
+  end
+end

data/lib/tapioca/gemfile.rb

@@ -28,7 +28,7 @@ module Tapioca
     sig { returns(T::Array[Gem]) }
     def dependencies
       @dependencies ||= begin
-        specs = definition.specs.to_a
+        specs = definition.locked_gems.specs.to_a
 
         definition
           .resolve
@@ -79,17 +79,18 @@ module Tapioca
       extend(T::Sig)
 
       IGNORED_GEMS = T.let(%w{
-        sorbet sorbet-static sorbet-runtime tapioca
+        sorbet sorbet-static sorbet-runtime
       }.freeze, T::Array[String])
 
       sig { returns(String) }
-      attr_reader :full_gem_path
+      attr_reader :full_gem_path, :version
 
       sig { params(spec: Spec).void }
       def initialize(spec)
         @spec = T.let(spec, Tapioca::Gemfile::Spec)
-        real_gem_path = File.realpath(@spec.full_gem_path.to_s)
+        real_gem_path = to_realpath(@spec.full_gem_path)
         @full_gem_path = T.let(real_gem_path, String)
+        @version = T.let(version_string, String)
       end
 
       sig { params(gemfile_dir: String).returns(T::Boolean) }
@@ -109,18 +110,52 @@ module Tapioca
         @spec.name
       end
 
-      sig { returns(::Gem::Version) }
-      def version
-        @spec.version
-      end
-
       sig { returns(String) }
       def rbi_file_name
         "#{name}@#{version}.rbi"
       end
 
+      sig { params(path: String).returns(T::Boolean) }
+      def contains_path?(path)
+        to_realpath(path).start_with?(full_gem_path) || has_parent_gemspec?(path)
+      end
+
       private
 
+      sig { returns(String) }
+      def version_string
+        version = @spec.version.to_s
+        version += "-#{@spec.source.revision}" if Bundler::Source::Git === @spec.source
+        version
+      end
+
+      sig { params(path: String).returns(T::Boolean) }
+      def has_parent_gemspec?(path)
+        # For some Git installed gems the location of the loaded file can
+        # be different from the gem path as indicated by the spec file
+        #
+        # To compensate for these cases, we walk up the directory hierarchy
+        # from the given file and try to match a <gem-name.gemspec> file in
+        # one of those folders to see if the path really belongs in the given gem
+        # or not.
+        return false unless Bundler::Source::Git === @spec.source
+        parent = Pathname.new(path)
+
+        until parent.root?
+          parent = parent.parent.expand_path
+          return true if parent.join("#{name}.gemspec").file?
+        end
+
+        false
+      end
+
+      sig { params(path: T.any(String, Pathname)).returns(String) }
+      def to_realpath(path)
+        path_string = path.to_s
+        path_string = File.realpath(path_string) if File.exist?(path_string)
+        path_string
+      end
+
       sig { returns(T::Boolean) }
       def gem_ignored?
         IGNORED_GEMS.include?(name)

data/lib/tapioca/generator.rb

@@ -8,41 +8,16 @@ module Tapioca
   class Generator < ::Thor::Shell::Color
     extend(T::Sig)
 
-    SORBET_CONFIG = "sorbet/config"
-    DEFAULT_POSTREQUIRE = "sorbet/tapioca/require.rb"
-    DEFAULT_OUTDIR = "sorbet/rbi/gems"
-    DEFAULT_OVERRIDES = T.let({
-      # ActiveSupport overrides some core methods with different signatures
-      # so we generate a typed: false RBI for it to suppress errors
-      "activesupport" => "false",
-    }.freeze, T::Hash[String, String])
-
-    sig { returns(Pathname) }
-    attr_reader :outdir
-    sig { returns(T.nilable(String)) }
-    attr_reader :prerequire
-    sig { returns(T.nilable(String)) }
-    attr_reader :postrequire
-    sig { returns(String) }
-    attr_reader :command
-    sig { returns(T::Hash[String, String]) }
-    attr_reader :typed_overrides
+    sig { returns(Config) }
+    attr_reader :config
 
     sig do
       params(
-        outdir: T.nilable(String),
-        prerequire: T.nilable(String),
-        postrequire: T.nilable(String),
-        command: T.nilable(String),
-        typed_overrides: T.nilable(T::Hash[String, String])
+        config: Config
       ).void
     end
-    def initialize(outdir: nil, prerequire: nil, postrequire: nil, command: nil, typed_overrides: nil)
-      @outdir = T.let(Pathname.new(outdir || DEFAULT_OUTDIR), Pathname)
-      @prerequire = T.let(prerequire, T.nilable(String))
-      @postrequire = T.let(postrequire || DEFAULT_POSTREQUIRE, T.nilable(String))
-      @command = T.let(command || default_command, String)
-      @typed_overrides = T.let(typed_overrides || {}, T::Hash[String, String])
+    def initialize(config)
+      @config = config
       @bundle = T.let(nil, T.nilable(Gemfile))
       @loader = T.let(nil, T.nilable(Loader))
       @compiler = T.let(nil, T.nilable(Compilers::SymbolTableCompiler))
@@ -55,14 +30,115 @@ module Tapioca
     def build_gem_rbis(gem_names)
       require_gem_file
 
-      gems_to_generate(gem_names).map do |gem|
-        say("Processing '#{gem.name}' gem:", :green)
-        indent do
-          compile_rbi(gem)
-          puts
+      gems_to_generate(gem_names)
+        .reject { |gem| config.exclude.include?(gem.name) }
+        .each do |gem|
+          say("Processing '#{gem.name}' gem:", :green)
+          indent do
+            compile_gem_rbi(gem)
+            puts
+          end
         end
+
+      say("All operations performed in working directory.", [:green, :bold])
+      say("Please review changes and commit them.", [:green, :bold])
+    end
+
+    sig { void }
+    def build_requires
+      requires_path = Config::DEFAULT_POSTREQUIRE
+      compiler = Compilers::RequiresCompiler.new(Config::SORBET_CONFIG)
+      name = set_color(requires_path, :yellow, :bold)
+      say("Compiling #{name}, this may take a few seconds... ")
+
+      rb_string = compiler.compile
+      if rb_string.empty?
+        say("Nothing to do", :green)
+        return
+      end
+
+      # Clean all existing requires before regenerating the list so we update
+      # it with the new one found in the client code and remove the old ones.
+      File.delete(requires_path) if File.exist?(requires_path)
+
+      content = String.new
+      content << rbi_header(
+        config.generate_command,
+        reason: "explicit gem requires",
+        strictness: "false"
+      )
+      content << rb_string
+
+      outdir = File.dirname(requires_path)
+      FileUtils.mkdir_p(outdir)
+      File.write(requires_path, content)
+
+      say("Done", :green)
+
+      say("All requires from this application have been written to #{name}.", [:green, :bold])
+      cmd = set_color("tapioca sync", :yellow, :bold)
+      say("Please review changes and commit them, then run #{cmd}.", [:green, :bold])
+    end
+
+    sig { void }
+    def build_todos
+      todos_path = config.todos_path
+      compiler = Compilers::TodosCompiler.new
+      name = set_color(todos_path, :yellow, :bold)
+      say("Compiling #{name}, this may take a few seconds... ")
+
+      # Clean all existing unresolved constants before regenerating the list
+      # so Sorbet won't grab them as already resolved.
+      File.delete(todos_path) if File.exist?(todos_path)
+
+      rbi_string = compiler.compile
+      if rbi_string.empty?
+        say("Nothing to do", :green)
+        return
+      end
+
+      content = String.new
+      content << rbi_header(
+        config.generate_command,
+        reason: "unresolved constants",
+        strictness: "false"
+      )
+      content << rbi_string
+      content << "\n"
+
+      outdir = File.dirname(todos_path)
+      FileUtils.mkdir_p(outdir)
+      File.write(todos_path, content)
+
+      say("Done", :green)
+
+      say("All unresolved constants have been written to #{name}.", [:green, :bold])
+      say("Please review changes and commit them.", [:green, :bold])
+    end
+
+    sig { params(requested_constants: T::Array[String]).void }
+    def build_dsl(requested_constants)
+      load_application(eager_load: requested_constants.empty?)
+      load_dsl_generators
+
+      say("Compiling DSL RBI files...")
+      say("")
+
+      compiler = Compilers::DslCompiler.new(
+        requested_constants: constantize(requested_constants),
+        requested_generators: config.generators,
+        error_handler: ->(error) {
+          say_error(error, :bold, :red)
+        }
+      )
+
+      compiler.run do |constant, contents|
+        compile_dsl_rbi(constant, contents)
       end
 
+      say("")
+      say("Done", :green)
+
       say("All operations performed in working directory.", [:green, :bold])
       say("Please review changes and commit them.", [:green, :bold])
     end
@@ -86,13 +162,10 @@ module Tapioca
 
     private
 
-    sig { returns(String) }
-    def default_command
-      command = File.basename($PROGRAM_NAME)
-      args = ARGV.join(" ")
-
-      "#{command} #{args}"
-    end
+    EMPTY_RBI_COMMENT = <<~CONTENT
+      # THIS IS AN EMPTY RBI FILE.
+      # see https://github.com/Shopify/tapioca/blob/master/README.md#manual-gem-requires
+    CONTENT
 
     sig { returns(Gemfile) }
     def bundle
@@ -112,42 +185,111 @@ module Tapioca
     sig { void }
     def require_gem_file
       say("Requiring all gems to prepare for compiling... ")
-      loader.load_bundle(prerequire, postrequire)
+      begin
+        loader.load_bundle(config.prerequire, config.postrequire)
+      rescue LoadError => e
+        explain_failed_require(config.postrequire, e)
+        exit(1)
+      end
       say(" Done", :green)
       puts
     end
 
+    sig { params(file: String, error: LoadError).void }
+    def explain_failed_require(file, error)
+      say_error("\n\nLoadError: #{error}", :bold, :red)
+      say_error("\nTapioca could not load all the gems required by your application.", :yellow)
+      say_error("If you populated ", :yellow)
+      say_error("#{file} ", :bold, :blue)
+      say_error("with ", :yellow)
+      say_error("tapioca require", :bold, :blue)
+      say_error("you should probably review it and remove the faulty line.", :yellow)
+    end
+
+    sig do
+      params(
+        message: String,
+        color: T.any(Symbol, T::Array[Symbol]),
+      ).void
+    end
+    def say_error(message = "", *color)
+      force_new_line = (message.to_s !~ /( |\t)\Z/)
+      buffer = prepare_message(*T.unsafe([message, *T.unsafe(color)]))
+      buffer << "\n" if force_new_line && !message.to_s.end_with?("\n")
+
+      stderr.print(buffer)
+      stderr.flush
+    end
+
+    sig { params(eager_load: T::Boolean).void }
+    def load_application(eager_load:)
+      say("Loading Rails application... ")
+
+      loader.load_rails(
+        environment_load: true,
+        eager_load: eager_load
+      )
+
+      say("Done", :green)
+    end
+
+    sig { void }
+    def load_dsl_generators
+      say("Loading DSL generator classes... ")
+
+      Dir.glob([
+        "#{__dir__}/compilers/dsl/*.rb",
+        "#{Config::TAPIOCA_PATH}/generators/**/*.rb",
+      ]).each do |generator|
+        require File.expand_path(generator)
+      end
+
+      say("Done", :green)
+    end
+
+    sig { params(constant_names: T::Array[String]).returns(T::Array[Module]) }
+    def constantize(constant_names)
+      constant_names.map do |name|
+        begin
+          name.constantize
+        rescue NameError
+          nil
+        end
+      end.compact
+    end
+
     sig { returns(T::Hash[String, String]) }
     def existing_rbis
-      @existing_rbis ||= Pathname.glob((Pathname.new(outdir) / "*@*.rbi").to_s)
-        .map { |f| f.basename(".*").to_s.split('@') }
+      @existing_rbis ||= Pathname.glob((config.outpath / "*@*.rbi").to_s)
+        .map { |f| T.cast(f.basename(".*").to_s.split('@', 2), [String, String]) }
         .to_h
     end
 
     sig { returns(T::Hash[String, String]) }
     def expected_rbis
       @expected_rbis ||= bundle.dependencies
+        .reject { |gem| config.exclude.include?(gem.name) }
         .map { |gem| [gem.name, gem.version.to_s] }
         .to_h
     end
 
     sig { params(gem_name: String, version: String).returns(Pathname) }
-    def rbi_filename(gem_name, version)
-      outdir / "#{gem_name}@#{version}.rbi"
+    def gem_rbi_filename(gem_name, version)
+      config.outpath / "#{gem_name}@#{version}.rbi"
     end
 
     sig { params(gem_name: String).returns(Pathname) }
     def existing_rbi(gem_name)
-      rbi_filename(gem_name, T.must(existing_rbis[gem_name]))
+      gem_rbi_filename(gem_name, T.must(existing_rbis[gem_name]))
     end
 
     sig { params(gem_name: String).returns(Pathname) }
     def expected_rbi(gem_name)
-      rbi_filename(gem_name, T.must(expected_rbis[gem_name]))
+      gem_rbi_filename(gem_name, T.must(expected_rbis[gem_name]))
     end
 
     sig { params(gem_name: String).returns(T::Boolean) }
-    def rbi_exists?(gem_name)
+    def gem_rbi_exists?(gem_name)
       existing_rbis.key?(gem_name)
     end
 
@@ -225,13 +367,13 @@ module Tapioca
           gems.each do |gem_name|
             filename = expected_rbi(gem_name)
 
-            if rbi_exists?(gem_name)
+            if gem_rbi_exists?(gem_name)
               old_filename = existing_rbi(gem_name)
               move(old_filename, filename) unless old_filename == filename
             end
 
             gem = T.must(bundle.gem(gem_name))
-            compile_rbi(gem)
+            compile_gem_rbi(gem)
             add(filename)
 
             puts
@@ -263,37 +405,73 @@ module Tapioca
       end
     end
 
-    sig { params(command: String, typed_sigil: String).returns(String) }
-    def rbi_header(command, typed_sigil)
-      <<~HEAD
-        # This file is autogenerated. Do not edit it by hand. Regenerate it with:
-        #   #{command}
+    sig { params(command: String, reason: T.nilable(String), strictness: T.nilable(String)).returns(String) }
+    def rbi_header(command, reason: nil, strictness: nil)
+      statement = <<~HEAD
+        # DO NOT EDIT MANUALLY
+        # This is an autogenerated file for #{reason}.
+        # Please instead update this file by running `#{command}`.
+      HEAD
 
-        # typed: #{typed_sigil}
+      sigil = <<~SIGIL if strictness
+        # typed: #{strictness}
+      SIGIL
 
-      HEAD
+      [statement, sigil].compact.join("\n").strip.concat("\n\n")
     end
 
     sig { params(gem: Gemfile::Gem).void }
-    def compile_rbi(gem)
+    def compile_gem_rbi(gem)
       compiler = Compilers::SymbolTableCompiler.new
       gem_name = set_color(gem.name, :yellow, :bold)
       say("Compiling #{gem_name}, this may take a few seconds... ")
 
-      typed_sigil = typed_overrides[gem.name] || DEFAULT_OVERRIDES[gem.name] || "true"
-
-      content = compiler.compile(gem)
-      content.prepend(rbi_header(command, typed_sigil))
-
-      FileUtils.mkdir_p(outdir)
-      filename = outdir / gem.rbi_file_name
+      strictness = config.typed_overrides[gem.name] || "true"
+      rbi_body_content = compiler.compile(gem)
+      content = String.new
+      content << rbi_header(
+        config.generate_command,
+        reason: "types exported from the `#{gem.name}` gem",
+        strictness: strictness
+      )
+
+      FileUtils.mkdir_p(config.outdir)
+      filename = config.outpath / gem.rbi_file_name
+
+      if rbi_body_content.strip.empty?
+        content << EMPTY_RBI_COMMENT
+        say("Done (empty output)", :yellow)
+      else
+        content << rbi_body_content
+        say("Done", :green)
+      end
       File.write(filename.to_s, content)
 
-      say("Done", :green)
-
-      Pathname.glob((outdir / "#{gem.name}@*.rbi").to_s) do |file|
+      Pathname.glob((config.outpath / "#{gem.name}@*.rbi").to_s) do |file|
         remove(file) unless file.basename.to_s == gem.rbi_file_name
       end
     end
+
+    sig { params(constant: Module, contents: String).void }
+    def compile_dsl_rbi(constant, contents)
+      return if contents.nil?
+
+      command = format(config.generate_command, constant.name)
+      constant_name = Module.instance_method(:name).bind(constant).call
+      rbi_name = constant_name.underscore + ".rbi"
+      filename = config.outpath / rbi_name
+
+      out = String.new
+      out << rbi_header(
+        command,
+        reason: "dynamic methods in `#{constant.name}`"
+      )
+      out << contents
+
+      FileUtils.mkdir_p(File.dirname(filename))
+      File.write(filename, out)
+      say("Wrote: ", [:green])
+      say(filename)
+    end
   end
 end