heroku_hatchet 8.0.0 → 8.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e66d5aea4901cc03c37bbf6d0f4b567e49f9c170d471ff461cc642956a195e9
-  data.tar.gz: 79fb324886b6a90828a753a9e93679db0386d1696f081efc33b05fef8d2fd5ab
+  metadata.gz: f4233a137df7685f1e411471db8c53a72c9c6e77e5b9a901d0d8bb1c0147f06a
+  data.tar.gz: ffd93abf69b7a46ccbdda5148710e237b9c41171abef209ea56f187d7a52c501
 SHA512:
-  metadata.gz: d337d285c3c35fc705e2f715bf209df425a1d36e908cba200610123b41eab488a59439109634c2456c148d6f877d93e9910becfde2606582803853bd9bce5913
-  data.tar.gz: 9c601a90d90d82782fd4fcdfca05ca1030738a1a0e93bd73c0cc42c3f60b91b7094b49b9128a3f4358ad4d161aceb2206850f90f155f2a1973529e705a00e345
+  metadata.gz: 61cc153424acbdf88c61b41ef9b4206fbb1e59c171ad30fded428d40bfe2edc728cdb2db72ca95526d1d5c5225d38eacacb6794b8c811b40988cac58480b0e74
+  data.tar.gz: 615e586767703298020a5d2a0172b9793a681a7cc681f45e6c6e42082e9db8c4a32f6e647fb006d8e9b7d8618b6c0dace7df6109cbca20e5e7fe0c34f991ed62

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## HEAD
 
+## 8.0.1
+
+- Bugfix: Lock and sleep and refresh API when duplicate app deletion detected (https://github.com/heroku/hatchet/pull/198)
+
 ## 8.0.0
 
 - Breaking change: Delete apps on teardown. Previously hatchet would delete apps lazily to help with debugging. This behavior allowed developers to inspect logs and `heroku run bash` in the event of an unexpected failure. In practice, it is rarely needed and causes accounts to retain apps indefinitely. Previously there was no cost to retaining applications, but now `basic` applications incur a charge. Change details:

data/bin/hatchet

@@ -123,7 +123,11 @@ class HatchetCLI < Thor
     when options[:older_than]
       minutes = options[:older_than].to_i
       puts "Destroying apps older than #{minutes}m"
-      reaper.destroy_older_apps(minutes: minutes)
+      reaper.destroy_older_apps(
+        minutes: minutes,
+        force_refresh: true,
+        on_conflict: :refresh_api_and_continue
+      )
       puts "Done"
     else
       raise "No flags given run `hatchet help destroy` for options"

data/lib/hatchet/app.rb

@@ -280,14 +280,31 @@ module Hatchet
     def create_app
       3.times.retry do
         begin
-          @reaper.destroy_older_apps
+          # Remove any obviously old apps first
+          # Try to use existing cache of apps to
+          # minimize API calls
+          @reaper.destroy_older_apps(
+            force_refresh: false,
+            on_conflict: :stop_if_under_limit,
+          )
           hash = { name: name, stack: stack }
           hash.delete_if { |k,v| v.nil? }
           result = heroku_api_create_app(hash)
           @heroku_id = result["id"]
         rescue => e
-          puts "Warning: Could not create app #{e.message}"
-          @reaper.clean_old_or_sleep
+          # If we can't create an app assume
+          # it might be due to resource constraints
+          #
+          # Try to delete existing apps
+          @reaper.destroy_older_apps(
+            force_refresh: true,
+            on_conflict: :stop_if_under_limit,
+          )
+          # If we're still not under the limit, sleep a bit
+          # retry later.
+          @reaper.sleep_if_over_limit(
+            reason: "Could not create app #{e.message}"
+          )
           raise e
         end
       end

data/lib/hatchet/reaper.rb

@@ -28,6 +28,13 @@ module Hatchet
     DEFAULT_REGEX = /^#{Regexp.escape(Hatchet::APP_PREFIX)}[a-f0-9]+/
     TTL_MINUTES = ENV.fetch("HATCHET_ALIVE_TTL_MINUTES", "7").to_i
 
+    # Protect against parallel deletion on the same machine
+    # via concurrent processes
+    #
+    # Does not protect against distributed systems on different
+    # machines trying to delete the same applications
+    MUTEX_FILE = File.open(File.join(Dir.tmpdir(), "hatchet_reaper_mutex"), File::CREAT)
+
     attr_accessor :io, :hatchet_app_limit
 
     def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit:  HATCHET_APP_LIMIT, initial_sleep: 10)
@@ -39,60 +46,127 @@ module Hatchet
       @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
     end
 
-    # Called when we need an app, but are over limit or
-    # if an exception has occured that was possibly triggered
-    # by apps being over limit
-    def clean_old_or_sleep
-      # Protect against parallel deletion of the same app on the same system
-      mutex_file = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
-      mutex_file.flock(File::LOCK_EX)
-
-      destroy_older_apps(force_refresh: true)
-
-      if @apps.length > @limit
+    def sleep_if_over_limit(reason: )
+      if @apps.length >= @limit
         age = AppAge.new(created_at: @apps.last["created_at"], ttl_minutes: TTL_MINUTES)
         @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
           io.puts <<-EOM.strip_heredoc
             WARNING: Hatchet app limit reached (#{@apps.length}/#{@limit})
-                     All known apps are younger than #{TTL_MINUTES} minutes
+            All known apps are younger than #{TTL_MINUTES} minutes.
+            Sleeping (#{sleep_for}s)
+
+            Reason: #{reason}
           EOM
 
           sleep(sleep_for)
         end
       end
-    ensure
-      mutex_file.close
     end
 
     # Destroys apps that are older than the given argument (expecting integer minutes)
-    def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?)
+    #
+    # This method might be running concurrently on multiple processes or multiple
+    # machines.
+    #
+    # When a duplicate destroy is detected we can move forward with a conflict strategy:
+    #
+    # - `:refresh_api_and_continue`: Sleep to see if another process will clean up everything for
+    #   us and then re-populate apps from the API and continue.
+    # - `:stop_if_under_limit`: Sleep to allow other processes to continue. Then if apps list
+    #   is under the limit, assume someone else is already cleaning up for us and that we're
+    #   good to move ahead to try to create an app. Otherwise if we're at or
+    #   over the limit sleep, refresh the app list, and continue attempting to delete apps.
+    def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?, on_conflict: :refresh_api_and_continue)
+      MUTEX_FILE.flock(File::LOCK_EX)
+
       refresh_app_list if force_refresh
 
-      @apps.each do |app|
+      while app = @apps.pop
         age = AppAge.new(created_at: app["created_at"], ttl_minutes: minutes)
-        if age.can_delete?
-          destroy_with_log(
-            name: app["name"],
-            id: app["id"],
-            reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
-          )
+        if !age.can_delete?
+          @apps.push(app)
+          break
+        else
+          begin
+            destroy_with_log(
+              id: app["id"],
+              name: app["name"],
+              reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
+            )
+          rescue AlreadyDeletedError => e
+            if handle_conflict(
+              strategy: on_conflict,
+              conflict_message: e.message,
+            ) == :stop
+              break
+            end
+          end
         end
-      rescue AlreadyDeletedError
-        # Ignore, keep going
       end
+    ensure
+      MUTEX_FILE.flock(File::LOCK_UN)
     end
 
     # No guardrails, will delete all apps that match the hatchet namespace
     def destroy_all(force_refresh: @apps.empty?)
+      MUTEX_FILE.flock(File::LOCK_EX)
+
       refresh_app_list if force_refresh
 
-      @apps.each do |app|
+      while app = @apps.pop
         begin
           destroy_with_log(name: app["name"], id: app["id"], reason: "destroy all")
-        rescue AlreadyDeletedError
-          # Ignore, keep going
+        rescue AlreadyDeletedError => e
+          handle_conflict(
+            conflict_message: e.message,
+            strategy: :refresh_api_and_continue
+          )
         end
       end
+    ensure
+      MUTEX_FILE.flock(File::LOCK_UN)
+    end
+
+    # Will sleep with backoff and emit a warning message
+    # returns :continue or :stop symbols
+    # :stop indicates execution should stop
+    private def handle_conflict(conflict_message:, strategy:)
+      message = String.new(<<-EOM.strip_heredoc)
+        WARNING: Possible race condition detected: #{conflict_message}
+        Hatchet app limit (#{@apps.length}/#{@limit}), using strategy #{strategy}
+      EOM
+
+      conflict_state = if :refresh_api_and_continue ==  strategy
+        message << "\nSleeping, refreshing app list, and continuing."
+        :continue
+      elsif :stop_if_under_limit == strategy && @apps.length >= @limit
+        message << "\nSleeping, refreshing app list, and continuing. Not under limit."
+        :continue
+      elsif :stop_if_under_limit == strategy
+        message << "\nHalting deletion of older apps. Under limit."
+        :stop
+      else
+        raise "No such strategy: #{strategy}, plese use :stop_if_under_limit or :refresh_api_and_continue"
+      end
+
+      @reaper_throttle.call(max_sleep: TTL_MINUTES) do |sleep_for|
+        io.puts <<-EOM.strip_heredoc
+          #{message}
+          Sleeping (#{sleep_for}s)
+        EOM
+
+        sleep(sleep_for)
+      end
+
+      case conflict_state
+      when :continue
+        refresh_app_list
+      when :stop
+      else
+        raise "Unknown state #{conflict_state}"
+      end
+
+      conflict_state
     end
 
     private def get_heroku_apps
@@ -117,15 +191,15 @@ module Hatchet
       body = e.response.body
       request_id = e.response.headers["Request-Id"]
       if body =~ /Couldn\'t find that app./
-        io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
-        raise AlreadyDeletedError.new
+        message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
+        raise AlreadyDeletedError.new(message)
       else
         raise e
       end
     rescue Excon::Error::Forbidden => e
       request_id = e.response.headers["Request-Id"]
-      io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
-      raise AlreadyDeletedError.new
+      message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
+      raise AlreadyDeletedError.new(message)
     end
   end
 end

data/lib/hatchet/version.rb

@@ -1,3 +1,3 @@
 module Hatchet
-  VERSION = "8.0.0"
+  VERSION = "8.0.1"
 end

data/spec/hatchet/app_spec.rb

@@ -43,7 +43,7 @@ describe "AppTest" do
 
     reaper = app.reaper
 
-    def reaper.clean_old_or_sleep; @app_exception_message = true; end
+    def reaper.destroy_older_apps(*args, **kwargs, &block); @app_exception_message = true; end
     def reaper.clean_old_was_called?; @app_exception_message; end
 
     expect {

data/spec/unit/reaper_spec.rb

@@ -18,7 +18,7 @@ describe "Reaper" do
   end
 
   describe "cycle" do
-    it "does not delete anything if under the limit" do
+    it "does not delete anything if no old apps" do
       reaper = Hatchet::Reaper.new(api_rate_limit: Object.new, hatchet_app_limit: 1, io: StringIO.new)
 
       def reaper.get_heroku_apps
@@ -29,7 +29,7 @@ describe "Reaper" do
       def reaper.check_get_heroku_apps_called; @called_get_heroku_apps ; end
       def reaper.reap_once; raise "should not be called"; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
 
       expect(reaper.check_get_heroku_apps_called).to be_truthy
     end
@@ -46,7 +46,7 @@ describe "Reaper" do
       end
       def reaper.destroy_called_with; @reaper_destroy_called_with; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
 
       expect(reaper.destroy_called_with).to eq({"name" => "hatchet-t-foo", "id" => 1})
     end
@@ -54,10 +54,10 @@ describe "Reaper" do
     it "sleeps, refreshes app list, and tries again when an old app is not past TTL" do
       warning = StringIO.new
       reaper = Hatchet::Reaper.new(
+        io: warning,
+        initial_sleep: 0,
         api_rate_limit: Object.new,
         hatchet_app_limit: 0,
-        initial_sleep: 0,
-        io: warning
       )
 
       def reaper.get_heroku_apps
@@ -74,7 +74,8 @@ describe "Reaper" do
 
       def reaper.get_slept_for_val; @_slept_for; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
+      reaper.sleep_if_over_limit(reason: "test")
 
       expect(reaper.get_slept_for_val).to eq(0)
       expect(reaper.destroy_called_with).to eq(nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heroku_hatchet
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.0.1
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-27 00:00:00.000000000 Z
+date: 2023-03-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: platform-api