heroku_hatchet 6.0.0 → 7.0.0

data/bin/hatchet

@@ -46,7 +46,7 @@ class HatchetCLI < Thor
       Threaded.later do
         commit = lock_hash[directory]
         directory = File.expand_path(directory)
-        if Dir[directory].present?
+        if !Dir[directory]&.empty?
           puts "== pulling '#{git_repo}' into '#{directory}'\n"
           pull(directory, git_repo)
         else
@@ -78,6 +78,8 @@ class HatchetCLI < Thor
 
         if lockfile_hash[directory] == "master"
           lock_hash[directory] = "master"
+        elsif lockfile_hash[directory] == "main"
+          lock_hash[directory] = "main"
         else
           commit = commit_at_directory(directory)
           lock_hash[directory] = commit
@@ -157,7 +159,7 @@ class HatchetCLI < Thor
   end
 
   def pull(path, git_repo, commit: false)
-    cmd("cd #{path} && git pull --rebase #{git_repo} master --quiet")
+    cmd("cd #{path} && git pull --rebase #{git_repo} --quiet")
   end
 
   def clone(path, git_repo, quiet: true)

data/hatchet.gemspec

@@ -18,11 +18,10 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "platform-api",  "3.0.0.pre.1 "
+  gem.add_dependency "platform-api",  "~> 3"
   gem.add_dependency "rrrretry",      "~> 1"
   gem.add_dependency "excon",         "~> 0"
   gem.add_dependency "thor",          "~> 0"
-  gem.add_dependency "repl_runner",   "~> 0.0.3"
   gem.add_dependency "threaded",      "~> 0"
 
   gem.add_development_dependency "rspec"

data/hatchet.json

@@ -12,6 +12,7 @@
  ],
  "lock": [
    "sharpstone/lock_fail",
-   "sharpstone/lock_fail_master"
+   "sharpstone/lock_fail_master",
+   "sharpstone/lock_fail_main"
  ]
 }

data/hatchet.lock

@@ -9,6 +9,8 @@
   - 6e642963acec0ff64af51bd6fba8db3c4176ed6e
 - - repo_fixtures/repos/lock/lock_fail
   - da748a59340be8b950e7bbbfb32077eb67d70c3c
+- - repo_fixtures/repos/lock/lock_fail_main
+  - main
 - - repo_fixtures/repos/lock/lock_fail_master
   - master
 - - repo_fixtures/repos/rails2/rails2blog

data/lib/hatchet.rb

@@ -1,4 +1,3 @@
-require 'active_support/core_ext/object/blank'
 require 'rrrretry'
 
 require 'json'
@@ -8,7 +7,7 @@ require 'stringio'
 require 'date'
 
 module Hatchet
-    APP_PREFIX = (ENV['HATCHET_APP_PREFIX'] || "hatchet-t-")
+  APP_PREFIX = (ENV['HATCHET_APP_PREFIX'] || "hatchet-t-")
 end
 
 require 'hatchet/version'

data/lib/hatchet/api_rate_limit.rb

@@ -1,14 +1,11 @@
-# Wraps platform-api and adds API rate limits
+# Legacy class
 #
-# Instead of:
-#
-#     platform_api.pipeline.create(name: @name)
-#
-# Use:
-#
-#     api_rate_limit = ApiRateLimit.new(platform_api)
-#     api_rate_limit.call.pipeline.create(name: @name)
+# Not needed since rate throttling went directly into the platform-api gem.
+# This class is effectively now a no-op
 #
+# It's being left in as it's interface was public and it's hard-ish to
+# deprecate/remove. Since it's so small there's not much value in removal
+# so it's probably fine to keep around for quite some time.
 class ApiRateLimit
   def initialize(platform_api)
     @platform_api = platform_api
@@ -16,14 +13,6 @@ class ApiRateLimit
     @called   = 0
   end
 
-
-  # Sleeps for progressively longer when api rate limit capacity
-  # is lower.
-  #
-  # Unfortunatley `@platform_api.rate_limit` is an extra API
-  # call, so by checking our limit, we also are using our limit 😬
-  # to partially mitigate this, only check capacity every 5
-  # api calls, or if the current capacity is under 1000
   def call
     # @called += 1
 

data/lib/hatchet/app.rb

@@ -9,14 +9,34 @@ module Hatchet
     HATCHET_BUILDPACK_BRANCH = -> { ENV['HATCHET_BUILDPACK_BRANCH'] || ENV['HEROKU_TEST_RUN_BRANCH'] || Hatchet.git_branch }
     BUILDPACK_URL = "https://github.com/heroku/heroku-buildpack-ruby.git"
 
-    attr_reader :name, :stack, :directory, :repo_name, :app_config, :buildpacks
-
-    class FailedDeploy < StandardError
-      def initialize(app, output)
-        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n" <<
-              " if this was expected add `allow_failure: true` to your deploy hash.\n" <<
-              "output:\n" <<
-              "#{output}"
+    attr_reader :name, :stack, :directory, :repo_name, :app_config, :buildpacks, :reaper
+
+    class FailedDeploy < StandardError; end
+
+    class FailedDeployError < FailedDeploy
+      attr_reader :output
+
+      def initialize(app, message, output: )
+        @output = output
+        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
+        msg << "#{message}\n"
+        msg << "output:\n"
+        msg << "#{output}"
+        super(msg)
+      end
+    end
+
+    class FailedReleaseError < FailedDeploy
+      attr_reader :output
+
+      def initialize(app, message, output: )
+        @output = output
+        msg = "Could not release '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
+        msg << "#{message}\n"
+        msg << "output:\n"
+        msg << "#{output}"
         super(msg)
       end
     end
@@ -34,6 +54,7 @@ module Hatchet
                    buildpacks: nil,
                    buildpack_url: nil,
                    before_deploy: nil,
+                   run_multi: ENV["HATCHET_RUN_MULTI"],
                    config: {}
                   )
       @repo_name     = repo_name
@@ -46,6 +67,12 @@ module Hatchet
       @buildpacks    = buildpack || buildpacks || buildpack_url || self.class.default_buildpack
       @buildpacks    = Array(@buildpacks)
       @buildpacks.map! {|b| b == :default ? self.class.default_buildpack : b}
+      @run_multi = run_multi
+
+      if run_multi && !ENV["HATCHET_EXPENSIVE_MODE"]
+        raise "You're attempting to enable `run_multi: true` mode, but have not enabled `HATCHET_EXPENSIVE_MODE=1` env var to verify you understand the risks"
+      end
+      @run_multi_array = []
       @already_in_dir = nil
       @app_is_setup = nil
 
@@ -127,6 +154,22 @@ module Hatchet
       else
         command = command.to_s
       end
+
+      heroku_command = build_heroku_command(command, options)
+
+      allow_run_multi! if @run_multi
+
+      output = ""
+
+      ShellThrottle.new(platform_api: @platform_api).call do |throttle|
+        output = `#{heroku_command}`
+        throw(:throttle) if output.match?(/reached the API rate limit/)
+      end
+
+      return output
+    end
+
+    private def build_heroku_command(command, options = {})
       command = command.shellescape unless options.delete(:raw)
 
       default_options = { "app" => name, "exit-code" => nil }
@@ -136,16 +179,77 @@ module Hatchet
         arg << "=#{v.to_s.shellescape}" unless v.nil? # nil means we include the option without an argument
         arg
       end.join(" ")
-      heroku_command = "heroku run #{heroku_options} -- #{command}"
 
-      if block_given?
-        STDERR.puts "Using App#run with a block is deprecated, support for ReplRunner is being removed.\n#{caller}"
-        # When we deprecated this we can get rid of the "cmd_type" from the method signature
-        require 'repl_runner'
-        ReplRunner.new(cmd_type, heroku_command, options).run(&block)
-      else
-        `#{heroku_command}`
+      "heroku run #{heroku_options} -- #{command}"
+    end
+
+    private def allow_run_multi!
+      raise "Must explicitly enable the `run_multi: true` option. This requires scaling up a dyno and is not free, it may incur charges on your account" unless @run_multi
+
+      @run_multi_is_setup ||= platform_api.formation.update(name, "web", {"size" => "Standard-1X"})
+    end
+
+
+    # Allows multiple commands to be run concurrently in the background.
+    #
+    # WARNING! Using the feature requres that the underlying app is not on the "free" Heroku
+    # tier. This requires scaling up the dyno which is not free. If an app is
+    # scaled up and left in that state it can incur large costs.
+    #
+    # Enabling this feature should be done with extreme caution.
+    #
+    # Example:
+    #
+    #    Hatchet::Runner.new("default_ruby", run_multi: true)
+    #      app.run_multi("ls") { |out| expect(out).to include("Gemfile") }
+    #      app.run_multi("ruby -v") { |out| expect(out).to include("ruby") }
+    #    end
+    #
+    # This example will run `heroku run ls` as well as `ruby -v` at the same time in the background.
+    # The return result will be yielded to the block after they finish running.
+    #
+    # Order of execution is not guaranteed.
+    #
+    # If you need to assert a command was successful, you can yield a second status object like this:
+    #
+    #    Hatchet::Runner.new("default_ruby", run_multi: true)
+    #      app.run_multi("ls") do |out, status|
+    #        expect(status.success?).to be_truthy
+    #        expect(out).to include("Gemfile")
+    #      end
+    #      app.run_multi("ruby -v") do |out, status|
+    #        expect(status.success?).to be_truthy
+    #        expect(out).to include("ruby")
+    #      end
+    #    end
+    def run_multi(command, options = {}, &block)
+      raise "Block required" if block.nil?
+      allow_run_multi!
+
+      run_thread = Thread.new do
+        heroku_command = build_heroku_command(command, options)
+
+        out = nil
+        status = nil
+        ShellThrottle.new(platform_api: @platform_api).call do |throttle|
+          out = `#{heroku_command}`
+          throw(:throttle) if output.match?(/reached the API rate limit/)
+          status = $?
+        end
+
+        yield out, status
+
+        # if block.arity == 1
+        #   block.call(out)
+        # else
+        #   block.call(out, status)
+        # end
       end
+      run_thread.abort_on_exception = true
+
+      @run_multi_array << run_thread
+
+      true
     end
 
     # set debug: true when creating app if you don't want it to be
@@ -162,24 +266,26 @@ module Hatchet
     alias :no_debug? :not_debugging?
 
     def deployed?
-      # !heroku.get_ps(name).body.detect {|ps| ps["process"].include?("web") }.nil?
       api_rate_limit.call.formation.list(name).detect {|ps| ps["type"] == "web"}
     end
 
     def create_app
       3.times.retry do
         begin
-          # heroku.post_app({ name: name, stack: stack }.delete_if {|k,v| v.nil? })
           hash = { name: name, stack: stack }
           hash.delete_if { |k,v| v.nil? }
-          api_rate_limit.call.app.create(hash)
+          heroku_api_create_app(hash)
         rescue => e
-          @reaper.cycle
+          @reaper.cycle(app_exception_message: e.message)
           raise e
         end
       end
     end
 
+    private def heroku_api_create_app(hash)
+      api_rate_limit.call.app.create(hash)
+    end
+
     def update_stack(stack_name)
       @stack = stack_name
       api_rate_limit.call.app.update(name, build_stack: @stack)
@@ -219,11 +325,15 @@ module Hatchet
 
     def teardown!
       return false unless @app_is_setup
-      if debugging?
-        puts "Debugging App:#{name}"
-        return false
+
+      if @run_multi_is_setup
+        @run_multi_array.map(&:join)
+        platform_api.formation.update(name, "web", {"size" => "free"})
       end
-      @reaper.cycle
+
+    ensure
+      @app_update_info = platform_api.app.update(name, { maintenance: true }) if @app_is_setup
+      @reaper.cycle if @app_is_setup
     end
 
     def in_directory(directory = self.directory)
@@ -265,10 +375,6 @@ module Hatchet
       end
     end
 
-    # creates a new app on heroku, "pushes" via anvil or git
-    # then yields to self so you can call self.run or
-    # self.deployed?
-    # Allow deploy failures on CI server by setting ENV['HATCHET_RETRIES']
     def deploy(&block)
       in_directory do
         self.setup!
@@ -276,7 +382,7 @@ module Hatchet
         block.call(self, api_rate_limit.call, output) if block_given?
       end
     ensure
-      self.teardown!
+      self.teardown! if block_given?
     end
 
     def push
@@ -346,6 +452,7 @@ module Hatchet
         test_run.wait!(&block)
       end
     ensure
+      teardown! if block_given?
       delete_pipeline(@pipeline_id) if @pipeline_id
       @pipeline_id = nil
     end
@@ -381,7 +488,6 @@ module Hatchet
     end
 
     def platform_api
-      puts "Deprecated: use `api_rate_limit.call` instead of platform_api"
       api_rate_limit
       return @platform_api
     end
@@ -433,3 +539,4 @@ module Hatchet
   end
 end
 
+require_relative 'shell_throttle.rb'

data/lib/hatchet/config.rb

@@ -46,7 +46,7 @@ module Hatchet
     def path_for_name(name)
       possible_paths = [repos[name.to_s], "repos/#{name}", name].compact
       path = possible_paths.detect do |path|
-        Dir[path].present?
+        !Dir[path]&.empty?
       end
       raise BadRepoName.new(name, possible_paths) if path.nil? || path.empty?
       path

data/lib/hatchet/git_app.rb

@@ -5,11 +5,37 @@ module Hatchet
       "https://git.heroku.com/#{name}.git"
     end
 
+
     def push_without_retry!
+      output = ""
+
+      ShellThrottle.new(platform_api: @platform_api).call do
+        output = git_push_heroku_yall
+      rescue FailedDeploy => e
+        if e.output.match?(/reached the API rate limit/)
+          throw(:throttle)
+        elsif @allow_failure
+          output = e.output
+        else
+          raise e
+        end
+      end
+
+      return output
+    end
+
+    private def git_push_heroku_yall
       output = `git push #{git_repo} master 2>&1`
+
       if !$?.success?
-        raise FailedDeploy.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}\n#{output}") unless @allow_failure
+        raise FailedDeployError.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}", output: output)
       end
+
+      releases = platform_api.release.list(name)
+      if releases.last["status"] == "failed"
+        raise FailedReleaseError.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}", output: output)
+      end
+
       return output
     end
   end

data/lib/hatchet/reaper.rb

@@ -1,88 +1,191 @@
 require 'tmpdir'
 
 module Hatchet
-  # Hatchet apps are useful after the tests run for debugging purposes
-  # the reaper is designed to allow the most recent apps to stay alive
-  # while keeping the total number of apps under the global Heroku limit.
-  # Any time you're worried about hitting the limit call @reaper.cycle
+  # This class lazilly deletes hatchet apps
   #
+  # When the reaper is called, it will check if the system has too many apps (Bassed off of HATCHET_APP_LIMIT), if so it will attempt
+  # to delete an app to free up capacity. The goal of lazilly deleting apps is to temporarilly keep
+  # apps around for debugging if they fail.
+  #
+  # When App#teardown! is called on an app it is marked as being in a "finished" state by turning
+  # on maintenance mode. The reaper will delete these in order (oldest first).
+  #
+  # If no apps are marked as being "finished" then the reaper will check to see if the oldest app
+  # has been alive for a long enough period for it's tests to finish (configured by HATCHET_ALIVE_TTL_MINUTES env var).
+  # If the "unfinished" app has been alive that long it will be deleted. If not, the system will sleep for a period of time
+  # in an attempt to allow other apps to move to be "finished".
+  #
+  # This class only limits and the number of "hatchet" apps on the system. Prevously there was a maximum of 100 apps on a
+  # Heroku account. Now a user can belong to multiple orgs and the total number of apps they have access to is no longer
+  # fixed at 100. Instead of hard coding a maximum limit, this failure mode is handled by forcing deletion of
+  # an app when app creation fails. In the future we may find a better way of detecting this failure mode
+  #
+  # Notes:
+  #
+  # - The class uses a file mutex so that multiple processes on the same machine do not attempt to run the
+  #   reaper at the same time.
+  # - AlreadyDeletedError will be raised if an app has already been deleted (possibly by another test run on
+  #   another machine). When this happens, the system will automatically attempt to reap another app.
   class Reaper
-    HEROKU_APP_LIMIT = Integer(ENV["HEROKU_APP_LIMIT"]  || 100) # the number of apps heroku allows you to keep
-    HATCHET_APP_LIMT = Integer(ENV["HATCHET_APP_LIMIT"] || 20)  # the number of apps hatchet keeps around
+    class AlreadyDeletedError < StandardError; end
+
+    HATCHET_APP_LIMIT = Integer(ENV["HATCHET_APP_LIMIT"] || 20)  # the number of apps hatchet keeps around
     DEFAULT_REGEX = /^#{Regexp.escape(Hatchet::APP_PREFIX)}[a-f0-9]+/
-    attr_accessor :apps
 
-    def initialize(api_rate_limit: , regex: DEFAULT_REGEX)
+    attr_accessor :io, :hatchet_app_limit
+
+    def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit:  HATCHET_APP_LIMIT, initial_sleep: 10)
       @api_rate_limit = api_rate_limit
-      @regex        = regex
+      @regex = regex
+      @io = io
+      @finished_hatchet_apps = []
+      @unfinished_hatchet_apps = []
+      @app_count = 0
+      @hatchet_app_limit = hatchet_app_limit
+      @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
+    end
+
+    def cycle(app_exception_message: false)
+      # Protect against parallel deletion of the same app on the same system
+      mutex_file = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
+      mutex_file.flock(File::LOCK_EX)
+
+      refresh_app_list if @finished_hatchet_apps.empty?
+
+      # To be safe try to delete an app even if we're not over the limit
+      # since the exception may have been caused by going over the maximum account limit
+      if app_exception_message
+          io.puts <<~EOM
+            WARNING: Running reaper due to exception on app
+                     #{stats_string}
+                     Exception: #{app_exception_message}
+          EOM
+        reap_once
+      end
+
+      while over_limit?
+        reap_once
+      end
+    ensure
+      mutex_file.close
     end
 
-    # Ascending order, oldest is last
-    def get_apps
-      apps          = @api_rate_limit.call.app.list.sort_by { |app| DateTime.parse(app["created_at"]) }.reverse
-      @app_count    = apps.count
-      @hatchet_apps = apps.select {|app| app["name"].match(@regex) }
+    def stats_string
+      "total_app_count: #{@app_count}, hatchet_app_count: #{hatchet_app_count}/#{HATCHET_APP_LIMIT}, finished: #{@finished_hatchet_apps.length}, unfinished: #{@unfinished_hatchet_apps.length}"
     end
 
-    def cycle
-      # we don't want multiple Hatchet processes (e.g. when using rspec-parallel) to delete apps at the same time
-      # this could otherwise result in race conditions in API causing errors other than 404s, making tests fail
-      mutex = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
-      mutex.flock(File::LOCK_EX)
+    def over_limit?
+      hatchet_app_count > hatchet_app_limit
+    end
 
-      # update list of apps once
+    # No guardrails, will delete all apps that match the hatchet namespace
+    def destroy_all
       get_apps
 
-      return unless over_limit?
-
-      while over_limit?
-        if @hatchet_apps.count > 1
-          # remove our own apps until we are below limit
-          destroy_oldest
-        else
-          puts "Warning: Reached Heroku app limit of #{HEROKU_APP_LIMIT}."
-          break
+      (@finished_hatchet_apps + @unfinished_hatchet_apps).each do |app|
+        begin
+          destroy_with_log(name: app["name"], id: app["id"])
+        rescue AlreadyDeletedError
+          # Ignore, keep going
         end
       end
+    end
 
-    # If the app is already deleted an exception
-    # will be raised, if the app cannot be found
-    # assume it is already deleted and try again
-    rescue Excon::Error::NotFound => e
-      body = e.response.body
-      if body =~ /Couldn\'t find that app./
-        puts "#{@message}, but looks like it was already deleted"
-        mutex.close # ensure only gets called on block exit and not on `retry`
-        retry
+    private def reap_once
+      refresh_app_list if @finished_hatchet_apps.empty?
+
+      if (app = @finished_hatchet_apps.pop)
+        destroy_with_log(name: app["name"], id: app["id"])
+      elsif (app = @unfinished_hatchet_apps.pop)
+        destroy_if_old_enough(app)
       end
-      raise e
-    ensure
-      # don't forget to close the mutex; this also releases our lock
-      mutex.close
+    rescue AlreadyDeletedError
+      retry
     end
 
-    def destroy_oldest
-      oldest = @hatchet_apps.pop
-      destroy_by_id(name: oldest["name"], id: oldest["id"], details: "Hatchet app limit: #{HATCHET_APP_LIMT}")
+    # Checks to see if the given app is older than the HATCHET_ALIVE_TTL_MINUTES
+    # if so, then the app is deleted, otherwise the reaper sleeps for a period of time after which
+    # It can try again to delete another app. The hope is that some apps will be marked as finished
+    # in that time
+    private def destroy_if_old_enough(app)
+      age = AppAge.new(
+        created_at: app["created_at"],
+        ttl_minutes: ENV.fetch("HATCHET_ALIVE_TTL_MINUTES", "7").to_i
+      )
+      if age.can_delete?
+        io.puts "WARNING: Destroying an app without maintenance mode on, app: #{app["name"]}, app_age: #{age.in_minutes} minutes"
+
+        destroy_with_log(name: app["name"], id: app["id"])
+      else
+        # We're not going to delete it yet, so put it back
+        @unfinished_hatchet_apps << app
+
+        # Sleep, try again later
+        @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
+          io.puts <<~EOM
+            WARNING: Attempting to destroy an app without maintenance mode on, but it is not old enough. app: #{app["name"]}, app_age: #{age.in_minutes} minutes
+                     This can happen if App#teardown! is not called on an application, which will leave it in an 'unfinished' state
+                     This can also happen if you're trying to run more tests concurrently than your currently set value for HATCHET_APP_COUNT
+                     Sleeping: #{sleep_for} seconds before trying to find another app to reap"
+                     #{stats_string}, HATCHET_ALIVE_TTL_MINUTES=#{age.ttl_minutes}
+          EOM
+
+          sleep(sleep_for)
+        end
+      end
     end
 
-    def destroy_all
-      get_apps
-      @hatchet_apps.each do |app|
-        destroy_by_id(name: app["name"], id: app["id"])
+    private def get_heroku_apps
+      @api_rate_limit.call.app.list
+    end
+
+    private def refresh_app_list
+      apps = get_heroku_apps.
+        map {|app| app["created_at"] = DateTime.parse(app["created_at"].to_s); app }.
+        sort_by { |app| app["created_at"] }.
+        reverse # Ascending order, oldest is last
+
+      @app_count = apps.length
+
+      @finished_hatchet_apps.clear
+      @unfinished_hatchet_apps.clear
+      apps.each do |app|
+        next unless app["name"].match(@regex)
+
+        if app["maintenance"]
+          @finished_hatchet_apps << app
+        else
+          @unfinished_hatchet_apps << app
+        end
       end
     end
 
-    def destroy_by_id(name:, id:, details: "")
-      @message = "Destroying #{name.inspect}: #{id}. #{details}"
-      puts @message
+    private def destroy_with_log(name:, id:)
+      message = "Destroying #{name.inspect}: #{id}, #{stats_string}"
+
       @api_rate_limit.call.app.delete(id)
-    end
 
-    private
+      io.puts message
+    rescue Excon::Error::NotFound => e
+      body = e.response.body
+      request_id = e.response.headers["Request-Id"]
+      if body =~ /Couldn\'t find that app./
+        io.puts "Duplicate destoy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
+        raise AlreadyDeletedError.new
+      else
+        raise e
+      end
+    rescue Excon::Error::Forbidden => e
+      request_id = e.response.headers["Request-Id"]
+      io.puts "Duplicate destoy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
+      raise AlreadyDeletedError.new
+    end
 
-    def over_limit?
-      @app_count > HEROKU_APP_LIMIT || @hatchet_apps.count > HATCHET_APP_LIMT
+    private def hatchet_app_count
+      @finished_hatchet_apps.length + @unfinished_hatchet_apps.length
     end
   end
 end
+
+require_relative "reaper/app_age"
+require_relative "reaper/reaper_throttle"