heroku_hatchet 5.0.2 → 7.1.1

data/lib/hatchet/config.rb

@@ -46,7 +46,7 @@ module Hatchet
     def path_for_name(name)
       possible_paths = [repos[name.to_s], "repos/#{name}", name].compact
       path = possible_paths.detect do |path|
-        Dir[path].present?
+        !Dir[path]&.empty?
       end
       raise BadRepoName.new(name, possible_paths) if path.nil? || path.empty?
       path

data/lib/hatchet/git_app.rb

@@ -5,11 +5,38 @@ module Hatchet
       "https://git.heroku.com/#{name}.git"
     end
 
+
     def push_without_retry!
+      output = ""
+
+      ShellThrottle.new(platform_api: @platform_api).call do
+        output = git_push_heroku_yall
+      rescue FailedDeploy => e
+        if e.output.match?(/reached the API rate limit/)
+          throw(:throttle)
+        elsif @allow_failure
+          output = e.output
+        else
+          raise e
+        end
+      end
+
+      return output
+    end
+
+    private def git_push_heroku_yall
       output = `git push #{git_repo} master 2>&1`
+
       if !$?.success?
-        raise FailedDeploy.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}\n#{output}") unless @allow_failure
+        raise FailedDeployError.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}", output: output)
       end
+
+      releases = platform_api.release.list(name)
+      if releases.last["status"] == "failed"
+        commit! # An empty commit allows us to deploy again
+        raise FailedReleaseError.new(self, "Buildpack: #{@buildpack.inspect}\nRepo: #{git_repo}", output: output)
+      end
+
       return output
     end
   end

data/lib/hatchet/reaper.rb

@@ -1,88 +1,191 @@
 require 'tmpdir'
 
 module Hatchet
-  # Hatchet apps are useful after the tests run for debugging purposes
-  # the reaper is designed to allow the most recent apps to stay alive
-  # while keeping the total number of apps under the global Heroku limit.
-  # Any time you're worried about hitting the limit call @reaper.cycle
+  # This class lazilly deletes hatchet apps
   #
+  # When the reaper is called, it will check if the system has too many apps (Bassed off of HATCHET_APP_LIMIT), if so it will attempt
+  # to delete an app to free up capacity. The goal of lazilly deleting apps is to temporarilly keep
+  # apps around for debugging if they fail.
+  #
+  # When App#teardown! is called on an app it is marked as being in a "finished" state by turning
+  # on maintenance mode. The reaper will delete these in order (oldest first).
+  #
+  # If no apps are marked as being "finished" then the reaper will check to see if the oldest app
+  # has been alive for a long enough period for it's tests to finish (configured by HATCHET_ALIVE_TTL_MINUTES env var).
+  # If the "unfinished" app has been alive that long it will be deleted. If not, the system will sleep for a period of time
+  # in an attempt to allow other apps to move to be "finished".
+  #
+  # This class only limits and the number of "hatchet" apps on the system. Prevously there was a maximum of 100 apps on a
+  # Heroku account. Now a user can belong to multiple orgs and the total number of apps they have access to is no longer
+  # fixed at 100. Instead of hard coding a maximum limit, this failure mode is handled by forcing deletion of
+  # an app when app creation fails. In the future we may find a better way of detecting this failure mode
+  #
+  # Notes:
+  #
+  # - The class uses a file mutex so that multiple processes on the same machine do not attempt to run the
+  #   reaper at the same time.
+  # - AlreadyDeletedError will be raised if an app has already been deleted (possibly by another test run on
+  #   another machine). When this happens, the system will automatically attempt to reap another app.
   class Reaper
-    HEROKU_APP_LIMIT = Integer(ENV["HEROKU_APP_LIMIT"]  || 100) # the number of apps heroku allows you to keep
-    HATCHET_APP_LIMT = Integer(ENV["HATCHET_APP_LIMIT"] || 20)  # the number of apps hatchet keeps around
+    class AlreadyDeletedError < StandardError; end
+
+    HATCHET_APP_LIMIT = Integer(ENV["HATCHET_APP_LIMIT"] || 20)  # the number of apps hatchet keeps around
     DEFAULT_REGEX = /^#{Regexp.escape(Hatchet::APP_PREFIX)}[a-f0-9]+/
-    attr_accessor :apps
 
-    def initialize(api_rate_limit: , regex: DEFAULT_REGEX)
+    attr_accessor :io, :hatchet_app_limit
+
+    def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit:  HATCHET_APP_LIMIT, initial_sleep: 10)
       @api_rate_limit = api_rate_limit
-      @regex        = regex
+      @regex = regex
+      @io = io
+      @finished_hatchet_apps = []
+      @unfinished_hatchet_apps = []
+      @app_count = 0
+      @hatchet_app_limit = hatchet_app_limit
+      @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
     end
 
-    # Ascending order, oldest is last
-    def get_apps
-      apps          = @api_rate_limit.call.app.list.sort_by { |app| DateTime.parse(app["created_at"]) }.reverse
-      @app_count    = apps.count
-      @hatchet_apps = apps.select {|app| app["name"].match(@regex) }
+    def cycle(app_exception_message: false)
+      # Protect against parallel deletion of the same app on the same system
+      mutex_file = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
+      mutex_file.flock(File::LOCK_EX)
+
+      refresh_app_list if @finished_hatchet_apps.empty?
+
+      # To be safe try to delete an app even if we're not over the limit
+      # since the exception may have been caused by going over the maximum account limit
+      if app_exception_message
+          io.puts <<~EOM
+            WARNING: Running reaper due to exception on app
+                     #{stats_string}
+                     Exception: #{app_exception_message}
+          EOM
+        reap_once
+      end
+
+      while over_limit?
+        reap_once
+      end
+    ensure
+      mutex_file.close
     end
 
-    def cycle
-      # we don't want multiple Hatchet processes (e.g. when using rspec-parallel) to delete apps at the same time
-      # this could otherwise result in race conditions in API causing errors other than 404s, making tests fail
-      mutex = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
-      mutex.flock(File::LOCK_EX)
+    def stats_string
+      "total_app_count: #{@app_count}, hatchet_app_count: #{hatchet_app_count}/#{HATCHET_APP_LIMIT}, finished: #{@finished_hatchet_apps.length}, unfinished: #{@unfinished_hatchet_apps.length}"
+    end
 
-      # update list of apps once
-      get_apps
+    def over_limit?
+      hatchet_app_count > hatchet_app_limit
+    end
 
-      return unless over_limit?
+    # No guardrails, will delete all apps that match the hatchet namespace
+    def destroy_all
+      refresh_app_list
 
-      while over_limit?
-        if @hatchet_apps.count > 1
-          # remove our own apps until we are below limit
-          destroy_oldest
-        else
-          puts "Warning: Reached Heroku app limit of #{HEROKU_APP_LIMIT}."
-          break
+      (@finished_hatchet_apps + @unfinished_hatchet_apps).each do |app|
+        begin
+          destroy_with_log(name: app["name"], id: app["id"])
+        rescue AlreadyDeletedError
+          # Ignore, keep going
         end
       end
+    end
 
-    # If the app is already deleted an exception
-    # will be raised, if the app cannot be found
-    # assume it is already deleted and try again
-    rescue Excon::Error::NotFound => e
-      body = e.response.body
-      if body =~ /Couldn\'t find that app./
-        puts "#{@message}, but looks like it was already deleted"
-        mutex.close # ensure only gets called on block exit and not on `retry`
-        retry
+    private def reap_once
+      refresh_app_list if @finished_hatchet_apps.empty?
+
+      if (app = @finished_hatchet_apps.pop)
+        destroy_with_log(name: app["name"], id: app["id"])
+      elsif (app = @unfinished_hatchet_apps.pop)
+        destroy_if_old_enough(app)
+      end
+    rescue AlreadyDeletedError
+      retry
+    end
+
+    # Checks to see if the given app is older than the HATCHET_ALIVE_TTL_MINUTES
+    # if so, then the app is deleted, otherwise the reaper sleeps for a period of time after which
+    # It can try again to delete another app. The hope is that some apps will be marked as finished
+    # in that time
+    private def destroy_if_old_enough(app)
+      age = AppAge.new(
+        created_at: app["created_at"],
+        ttl_minutes: ENV.fetch("HATCHET_ALIVE_TTL_MINUTES", "7").to_i
+      )
+      if age.can_delete?
+        io.puts "WARNING: Destroying an app without maintenance mode on, app: #{app["name"]}, app_age: #{age.in_minutes} minutes"
+
+        destroy_with_log(name: app["name"], id: app["id"])
+      else
+        # We're not going to delete it yet, so put it back
+        @unfinished_hatchet_apps << app
+
+        # Sleep, try again later
+        @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
+          io.puts <<~EOM
+            WARNING: Attempting to destroy an app without maintenance mode on, but it is not old enough. app: #{app["name"]}, app_age: #{age.in_minutes} minutes
+                     This can happen if App#teardown! is not called on an application, which will leave it in an 'unfinished' state
+                     This can also happen if you're trying to run more tests concurrently than your currently set value for HATCHET_APP_COUNT
+                     Sleeping: #{sleep_for} seconds before trying to find another app to reap"
+                     #{stats_string}, HATCHET_ALIVE_TTL_MINUTES=#{age.ttl_minutes}
+          EOM
+
+          sleep(sleep_for)
+        end
       end
-      raise e
-    ensure
-      # don't forget to close the mutex; this also releases our lock
-      mutex.close
     end
 
-    def destroy_oldest
-      oldest = @hatchet_apps.pop
-      destroy_by_id(name: oldest["name"], id: oldest["id"], details: "Hatchet app limit: #{HATCHET_APP_LIMT}")
+    private def get_heroku_apps
+      @api_rate_limit.call.app.list
     end
 
-    def destroy_all
-      get_apps
-      @hatchet_apps.each do |app|
-        destroy_by_id(name: app["name"], id: app["id"])
+    private def refresh_app_list
+      apps = get_heroku_apps.
+        map {|app| app["created_at"] = DateTime.parse(app["created_at"].to_s); app }.
+        sort_by { |app| app["created_at"] }.
+        reverse # Ascending order, oldest is last
+
+      @app_count = apps.length
+
+      @finished_hatchet_apps.clear
+      @unfinished_hatchet_apps.clear
+      apps.each do |app|
+        next unless app["name"].match(@regex)
+
+        if app["maintenance"]
+          @finished_hatchet_apps << app
+        else
+          @unfinished_hatchet_apps << app
+        end
       end
     end
 
-    def destroy_by_id(name:, id:, details: "")
-      @message = "Destroying #{name.inspect}: #{id}. #{details}"
-      puts @message
+    private def destroy_with_log(name:, id:)
+      message = "Destroying #{name.inspect}: #{id}, #{stats_string}"
+
       @api_rate_limit.call.app.delete(id)
-    end
 
-    private
+      io.puts message
+    rescue Excon::Error::NotFound => e
+      body = e.response.body
+      request_id = e.response.headers["Request-Id"]
+      if body =~ /Couldn\'t find that app./
+        io.puts "Duplicate destoy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
+        raise AlreadyDeletedError.new
+      else
+        raise e
+      end
+    rescue Excon::Error::Forbidden => e
+      request_id = e.response.headers["Request-Id"]
+      io.puts "Duplicate destoy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
+      raise AlreadyDeletedError.new
+    end
 
-    def over_limit?
-      @app_count > HEROKU_APP_LIMIT || @hatchet_apps.count > HATCHET_APP_LIMT
+    private def hatchet_app_count
+      @finished_hatchet_apps.length + @unfinished_hatchet_apps.length
     end
   end
 end
+
+require_relative "reaper/app_age"
+require_relative "reaper/reaper_throttle"

data/lib/hatchet/reaper/app_age.rb

@@ -0,0 +1,49 @@
+module Hatchet
+  class Reaper
+    # Class for figuring out how old a given time is relative to another time
+    #
+    # Expects inputs as a DateTime instance
+    #
+    # Example:
+    #
+    #   time_now = DateTime.parse("2020-07-28T14:40:00Z")
+    #   age = AppAge.new(created_at: DateTIme.parse("2020-07-28T14:40:00Z"), time_now: time_now, ttl_minutes: 1)
+    #   age.in_minutes => 0.0
+    #   age.too_young_to_die? # => true
+    #   age.can_delete? # => false
+    #   age.sleep_for_ttl #=> 60
+    class AppAge
+      SECONDS_IN_A_DAY = 24 * 60 * 60
+
+      attr_reader :ttl_minutes
+
+      def initialize(created_at:, ttl_minutes:, time_now: DateTime.now.new_offset(0))
+        @seconds_ago = date_time_diff_in_seconds(time_now, created_at)
+        @ttl_minutes = ttl_minutes
+        @ttl_seconds = ttl_minutes * 60
+      end
+
+      def date_time_diff_in_seconds(now, whence)
+        (now - whence) * SECONDS_IN_A_DAY
+      end
+
+      def too_young_to_die?
+        !can_delete?
+      end
+
+      def can_delete?
+        @seconds_ago > @ttl_seconds
+      end
+
+      def sleep_for_ttl
+        return 0 if can_delete?
+
+        @ttl_seconds - @seconds_ago
+      end
+
+      def in_minutes
+        (@seconds_ago / 60.0).round(2)
+      end
+    end
+  end
+end

data/lib/hatchet/reaper/reaper_throttle.rb

@@ -0,0 +1,55 @@
+module Hatchet
+  class Reaper
+    # This class retains and increments a sleep value between executions
+    #
+    # Every time we pause, we increase the duration of the pause 2x. If we
+    # do not sleep for long enough then we will burn API requests that we don't need to make.
+    #
+    # To help prevent sleeping for too long, the reaper will sleep for a maximum amount of time
+    # equal to the age_sleep_for_ttl. If that happens, it's likely a fairly large value and the
+    # internal incremental value can be reset
+    #
+    # Example:
+    #
+    #   reaper_throttle = ReaperThrottle.new(initial_sleep: 2)
+    #   reaper_throttle.call(max_sleep: 5) do |sleep_for|
+    #     puts sleep_for # => 2
+    #   end
+    #   reaper_throttle.call(max_sleep: 5) do |sleep_for|
+    #     puts sleep_for # => 4
+    #   end
+    #   reaper_throttle.call(max_sleep: 5) do |sleep_for|
+    #     puts sleep_for # => 5
+    #   end
+    #
+    #   # The throttle is now reset since it hit the max_sleep value
+    #
+    #   reaper_throttle.call(max_sleep: 5) do |sleep_for|
+    #     puts sleep_for # => 2
+    #   end
+    class ReaperThrottle
+      def initialize(initial_sleep: )
+        @initial_sleep = initial_sleep
+        @sleep_for = @initial_sleep
+      end
+
+      def call(max_sleep: )
+        raise "Must call with a block" unless block_given?
+
+        sleep_for = [@sleep_for, max_sleep].min
+
+        yield sleep_for
+
+        if sleep_for < @sleep_for
+          reset!
+        else
+          @sleep_for *= 2
+        end
+      end
+
+      def reset!
+        @sleep_for = @initial_sleep
+      end
+    end
+  end
+end

data/lib/hatchet/shell_throttle.rb

@@ -0,0 +1,71 @@
+module Hatchet
+  # A class for throttling non-http resources
+  #
+  # Non-http calls can be rate-limited for example shell calls to `heroku run ` and `git push heroku`
+  # this class provides an easy interface to leverage the rate throttling behavior baked into `PlatformAPI`
+  # for calls things that do not have a real associated web request
+  #
+  # Example:
+  #
+  #   output = ""
+  #   ShellThrottle.new(platform_api: @platform_api).call
+  #     output = `git push heroku main`
+  #     throw(:throttle) if output.match?(/reached the API rate limit/)
+  #   end
+  #   puts output
+  #
+  # In this example `git push heroku main` will retry and backoff until the output no longer matches `reached the API rate limit`.
+  #
+  class ShellThrottle
+    def initialize(platform_api: )
+      @platform_api = platform_api
+    end
+
+    def call
+      out = nil
+      PlatformAPI.rate_throttle.call do
+        catch(:throttle) do
+          out = yield
+          return
+        end
+
+        try_again
+      end
+      return out
+    end
+
+    private def success
+      FakeResponse.new(status: 200, remaining: remaining)
+    end
+
+    private def try_again
+      FakeResponse.new(status: 429, remaining: remaining)
+    end
+
+    private def remaining
+      @platform_api.rate_limit.info["remaining"]
+    end
+
+
+    # Helper class to be used along with the PlatformAPI.rate_throttle interface
+    # that expects a response object
+    #
+    # Example:
+    #
+    #   FakeResponse.new(status: 200, remaining: 2).status #=> 200
+    #   FakeResponse.new(status: 200, remaining: 2).headers["RateLimit-Remaining"] #=> 2
+    class FakeResponse
+      attr_reader :status, :headers
+
+      def initialize(status:, remaining: )
+        @status = status
+
+        @headers = {
+          "RateLimit-Remaining" => remaining,
+          "RateLimit-Multiplier" => 1,
+          "Content-Type" => "text/plain".freeze
+        }
+      end
+    end
+  end
+end