heroku_hatchet 8.0.0 → 8.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e66d5aea4901cc03c37bbf6d0f4b567e49f9c170d471ff461cc642956a195e9
-  data.tar.gz: 79fb324886b6a90828a753a9e93679db0386d1696f081efc33b05fef8d2fd5ab
+  metadata.gz: c72d006f643f893a0347e7c7ce5fae2e128d4a94fea501d957afb6738cc9374f
+  data.tar.gz: a507cd401042cf6e245b63824b9cbb30b79aa56541bc604f2c92b54d23413cbd
 SHA512:
-  metadata.gz: d337d285c3c35fc705e2f715bf209df425a1d36e908cba200610123b41eab488a59439109634c2456c148d6f877d93e9910becfde2606582803853bd9bce5913
-  data.tar.gz: 9c601a90d90d82782fd4fcdfca05ca1030738a1a0e93bd73c0cc42c3f60b91b7094b49b9128a3f4358ad4d161aceb2206850f90f155f2a1973529e705a00e345
+  metadata.gz: 6ba49829178ac76fe6a62faf32afbef57a828dc4fd390d50887f2b2c50f30a8359ad6f9639ad34b258a2594a885333e8a704b59f93e3bdf80cb87ce897f5a4c1
+  data.tar.gz: a54f637129f9b57f296f405e374082573b584b3c960bab387723f19f87bf8ee63562022a94ee08aa9e75bf3badffd63fdeec805897b71db30e15b74aaf5384c6

data/.github/workflows/hatchet_app_cleaner.yml

@@ -0,0 +1,31 @@
+name: Hatchet app cleaner
+
+on:
+  schedule:
+    # Daily at 6am UTC.
+    - cron: "0 6 * * *"
+  # Allow the workflow to be manually triggered too.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  hatchet-app-cleaner:
+    runs-on: ubuntu-latest
+    env:
+      HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
+      HEROKU_API_USER: ${{ secrets.HEROKU_API_USER }}
+      HEROKU_DISABLE_AUTOUPDATE: 1
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install Ruby and dependencies
+        uses: ruby/setup-ruby@v1
+        with:
+          bundler-cache: true
+          ruby-version: "3.1"
+      - name: Run Hatchet destroy
+        # Only apps older than 10 minutes are destroyed, to ensure that any
+        # in progress CI runs are not interrupted.
+        run: bundle exec hatchet destroy --older-than 10

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 ## HEAD
 
+## 8.0.2
+
+- Bugfix: Allow nested deploy blocks with new teardown logic (https://github.com/heroku/hatchet/pull/201)
+
+## 8.0.1
+
+- Bugfix: Lock and sleep and refresh API when duplicate app deletion detected (https://github.com/heroku/hatchet/pull/198)
+
 ## 8.0.0
 
 - Breaking change: Delete apps on teardown. Previously hatchet would delete apps lazily to help with debugging. This behavior allowed developers to inspect logs and `heroku run bash` in the event of an unexpected failure. In practice, it is rarely needed and causes accounts to retain apps indefinitely. Previously there was no cost to retaining applications, but now `basic` applications incur a charge. Change details:

data/bin/hatchet

@@ -123,7 +123,11 @@ class HatchetCLI < Thor
     when options[:older_than]
       minutes = options[:older_than].to_i
       puts "Destroying apps older than #{minutes}m"
-      reaper.destroy_older_apps(minutes: minutes)
+      reaper.destroy_older_apps(
+        minutes: minutes,
+        force_refresh: true,
+        on_conflict: :refresh_api_and_continue
+      )
       puts "Done"
     else
       raise "No flags given run `hatchet help destroy` for options"

data/lib/hatchet/app.rb

@@ -77,6 +77,7 @@ module Hatchet
       @buildpacks.map! {|b| b == :default ? self.class.default_buildpack : b}
       @run_multi = run_multi
       @max_retries_count = retries
+      @outer_deploy_block = nil
 
       if run_multi && !ENV["HATCHET_EXPENSIVE_MODE"]
         raise "You're attempting to enable `run_multi: true` mode, but have not enabled `HATCHET_EXPENSIVE_MODE=1` env var to verify you understand the risks"
@@ -280,14 +281,31 @@ module Hatchet
     def create_app
       3.times.retry do
         begin
-          @reaper.destroy_older_apps
+          # Remove any obviously old apps first
+          # Try to use existing cache of apps to
+          # minimize API calls
+          @reaper.destroy_older_apps(
+            force_refresh: false,
+            on_conflict: :stop_if_under_limit,
+          )
           hash = { name: name, stack: stack }
           hash.delete_if { |k,v| v.nil? }
           result = heroku_api_create_app(hash)
           @heroku_id = result["id"]
         rescue => e
-          puts "Warning: Could not create app #{e.message}"
-          @reaper.clean_old_or_sleep
+          # If we can't create an app assume
+          # it might be due to resource constraints
+          #
+          # Try to delete existing apps
+          @reaper.destroy_older_apps(
+            force_refresh: true,
+            on_conflict: :stop_if_under_limit,
+          )
+          # If we're still not under the limit, sleep a bit
+          # retry later.
+          @reaper.sleep_if_over_limit(
+            reason: "Could not create app #{e.message}"
+          )
           raise e
         end
       end
@@ -407,13 +425,14 @@ module Hatchet
     def deploy(&block)
       in_directory do
         annotate_failures do
+          @outer_deploy_block ||= block # deploy! can be called multiple times. Only teardown once
           in_dir_setup!
-          self.push_with_retry!
+          push_with_retry!
           block.call(self, api_rate_limit.call, output) if block_given?
         end
       end
     ensure
-      self.teardown! if block_given?
+      self.teardown! if block_given? && @outer_deploy_block == block
     end
 
     def push

data/lib/hatchet/reaper.rb

@@ -28,6 +28,13 @@ module Hatchet
     DEFAULT_REGEX = /^#{Regexp.escape(Hatchet::APP_PREFIX)}[a-f0-9]+/
     TTL_MINUTES = ENV.fetch("HATCHET_ALIVE_TTL_MINUTES", "7").to_i
 
+    # Protect against parallel deletion on the same machine
+    # via concurrent processes
+    #
+    # Does not protect against distributed systems on different
+    # machines trying to delete the same applications
+    MUTEX_FILE = File.open(File.join(Dir.tmpdir(), "hatchet_reaper_mutex"), File::CREAT)
+
     attr_accessor :io, :hatchet_app_limit
 
     def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit:  HATCHET_APP_LIMIT, initial_sleep: 10)
@@ -39,60 +46,127 @@ module Hatchet
       @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
     end
 
-    # Called when we need an app, but are over limit or
-    # if an exception has occured that was possibly triggered
-    # by apps being over limit
-    def clean_old_or_sleep
-      # Protect against parallel deletion of the same app on the same system
-      mutex_file = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
-      mutex_file.flock(File::LOCK_EX)
-
-      destroy_older_apps(force_refresh: true)
-
-      if @apps.length > @limit
+    def sleep_if_over_limit(reason: )
+      if @apps.length >= @limit
         age = AppAge.new(created_at: @apps.last["created_at"], ttl_minutes: TTL_MINUTES)
         @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
           io.puts <<-EOM.strip_heredoc
             WARNING: Hatchet app limit reached (#{@apps.length}/#{@limit})
-                     All known apps are younger than #{TTL_MINUTES} minutes
+            All known apps are younger than #{TTL_MINUTES} minutes.
+            Sleeping (#{sleep_for}s)
+
+            Reason: #{reason}
           EOM
 
           sleep(sleep_for)
         end
       end
-    ensure
-      mutex_file.close
     end
 
     # Destroys apps that are older than the given argument (expecting integer minutes)
-    def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?)
+    #
+    # This method might be running concurrently on multiple processes or multiple
+    # machines.
+    #
+    # When a duplicate destroy is detected we can move forward with a conflict strategy:
+    #
+    # - `:refresh_api_and_continue`: Sleep to see if another process will clean up everything for
+    #   us and then re-populate apps from the API and continue.
+    # - `:stop_if_under_limit`: Sleep to allow other processes to continue. Then if apps list
+    #   is under the limit, assume someone else is already cleaning up for us and that we're
+    #   good to move ahead to try to create an app. Otherwise if we're at or
+    #   over the limit sleep, refresh the app list, and continue attempting to delete apps.
+    def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?, on_conflict: :refresh_api_and_continue)
+      MUTEX_FILE.flock(File::LOCK_EX)
+
       refresh_app_list if force_refresh
 
-      @apps.each do |app|
+      while app = @apps.pop
         age = AppAge.new(created_at: app["created_at"], ttl_minutes: minutes)
-        if age.can_delete?
-          destroy_with_log(
-            name: app["name"],
-            id: app["id"],
-            reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
-          )
+        if !age.can_delete?
+          @apps.push(app)
+          break
+        else
+          begin
+            destroy_with_log(
+              id: app["id"],
+              name: app["name"],
+              reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
+            )
+          rescue AlreadyDeletedError => e
+            if handle_conflict(
+              strategy: on_conflict,
+              conflict_message: e.message,
+            ) == :stop
+              break
+            end
+          end
         end
-      rescue AlreadyDeletedError
-        # Ignore, keep going
       end
+    ensure
+      MUTEX_FILE.flock(File::LOCK_UN)
     end
 
     # No guardrails, will delete all apps that match the hatchet namespace
     def destroy_all(force_refresh: @apps.empty?)
+      MUTEX_FILE.flock(File::LOCK_EX)
+
       refresh_app_list if force_refresh
 
-      @apps.each do |app|
+      while app = @apps.pop
         begin
           destroy_with_log(name: app["name"], id: app["id"], reason: "destroy all")
-        rescue AlreadyDeletedError
-          # Ignore, keep going
+        rescue AlreadyDeletedError => e
+          handle_conflict(
+            conflict_message: e.message,
+            strategy: :refresh_api_and_continue
+          )
         end
       end
+    ensure
+      MUTEX_FILE.flock(File::LOCK_UN)
+    end
+
+    # Will sleep with backoff and emit a warning message
+    # returns :continue or :stop symbols
+    # :stop indicates execution should stop
+    private def handle_conflict(conflict_message:, strategy:)
+      message = String.new(<<-EOM.strip_heredoc)
+        WARNING: Possible race condition detected: #{conflict_message}
+        Hatchet app limit (#{@apps.length}/#{@limit}), using strategy #{strategy}
+      EOM
+
+      conflict_state = if :refresh_api_and_continue ==  strategy
+        message << "\nSleeping, refreshing app list, and continuing."
+        :continue
+      elsif :stop_if_under_limit == strategy && @apps.length >= @limit
+        message << "\nSleeping, refreshing app list, and continuing. Not under limit."
+        :continue
+      elsif :stop_if_under_limit == strategy
+        message << "\nHalting deletion of older apps. Under limit."
+        :stop
+      else
+        raise "No such strategy: #{strategy}, plese use :stop_if_under_limit or :refresh_api_and_continue"
+      end
+
+      @reaper_throttle.call(max_sleep: TTL_MINUTES) do |sleep_for|
+        io.puts <<-EOM.strip_heredoc
+          #{message}
+          Sleeping (#{sleep_for}s)
+        EOM
+
+        sleep(sleep_for)
+      end
+
+      case conflict_state
+      when :continue
+        refresh_app_list
+      when :stop
+      else
+        raise "Unknown state #{conflict_state}"
+      end
+
+      conflict_state
     end
 
     private def get_heroku_apps
@@ -117,15 +191,15 @@ module Hatchet
       body = e.response.body
       request_id = e.response.headers["Request-Id"]
       if body =~ /Couldn\'t find that app./
-        io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
-        raise AlreadyDeletedError.new
+        message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
+        raise AlreadyDeletedError.new(message)
       else
         raise e
       end
     rescue Excon::Error::Forbidden => e
       request_id = e.response.headers["Request-Id"]
-      io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
-      raise AlreadyDeletedError.new
+      message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
+      raise AlreadyDeletedError.new(message)
     end
   end
 end

data/lib/hatchet/version.rb

@@ -1,3 +1,3 @@
 module Hatchet
-  VERSION = "8.0.0"
+  VERSION = "8.0.2"
 end

data/spec/hatchet/app_spec.rb

@@ -1,20 +1,6 @@
 require("spec_helper")
 
 describe "AppTest" do
-  it "annotates rspec expectation failures" do
-    app = Hatchet::Runner.new("default_ruby")
-    error = nil
-    begin
-      app.annotate_failures do
-        expect(true).to eq(false)
-      end
-    rescue RSpec::Expectations::ExpectationNotMetError => e
-      error = e
-    end
-
-    expect(error.message).to include(app.name)
-  end
-
   it "does not modify local files by mistake" do
     Dir.mktmpdir do |dir_1|
       app = Hatchet::Runner.new(dir_1)
@@ -29,7 +15,6 @@ describe "AppTest" do
         entries_array -= ["..", ".", "foo.txt"]
         expect(entries_array).to be_empty
 
-
         entries_array = Dir.entries(dir_1)
         entries_array -= ["..", ".", "foo.txt"]
         expect(entries_array).to be_empty
@@ -37,33 +22,8 @@ describe "AppTest" do
     end
   end
 
-  it "calls reaper if cannot create an app" do
-    app = Hatchet::App.new("default_ruby", buildpacks: [:default])
-    def app.heroku_api_create_app(*args); raise StandardError.new("made you look"); end
-
-    reaper = app.reaper
-
-    def reaper.clean_old_or_sleep; @app_exception_message = true; end
-    def reaper.clean_old_was_called?; @app_exception_message; end
-
-    expect {
-      app.create_app
-    }.to raise_error("made you look")
-
-    expect(reaper.clean_old_was_called?).to be_truthy
-  end
-
-  it "app with default" do
-    app = Hatchet::App.new("default_ruby", buildpacks: [:default])
-    expect(app.buildpacks.first).to match("https://github.com/heroku/heroku-buildpack-ruby")
-  end
-
-  it "default_buildpack is only computed once" do
-    expect(Hatchet::App.default_buildpack.object_id).to eq(Hatchet::App.default_buildpack.object_id)
-  end
-
   it "create app with stack" do
-    stack = "heroku-18"
+    stack = "heroku-20"
     app = Hatchet::App.new("default_ruby", stack: stack)
     app.create_app
     expect(app.platform_api.app.info(app.name)["build_stack"]["name"]).to eq(stack)

data/spec/unit/app_spec.rb

@@ -0,0 +1,65 @@
+require "spec_helper"
+
+# Tests in this file do not deploy to Heroku
+describe "App unit tests" do
+  it "annotates rspec expectation failures" do
+    app = Hatchet::Runner.new("default_ruby")
+    error = nil
+    begin
+      app.annotate_failures do
+        expect(true).to eq(false)
+      end
+    rescue RSpec::Expectations::ExpectationNotMetError => e
+      error = e
+    end
+
+    expect(error.message).to include(app.name)
+  end
+
+  it "calls reaper if cannot create an app" do
+    app = Hatchet::App.new("default_ruby", buildpacks: [:default])
+    def app.heroku_api_create_app(*args); raise StandardError.new("made you look"); end
+
+    reaper = app.reaper
+
+    def reaper.destroy_older_apps(*args, **kwargs, &block); @app_exception_message = true; end
+    def reaper.clean_old_was_called?; @app_exception_message; end
+
+    expect {
+      app.create_app
+    }.to raise_error("made you look")
+
+    expect(reaper.clean_old_was_called?).to be_truthy
+  end
+
+  it "app with default" do
+    app = Hatchet::App.new("default_ruby", buildpacks: [:default])
+    expect(app.buildpacks.first).to match("https://github.com/heroku/heroku-buildpack-ruby")
+  end
+
+  it "default_buildpack is only computed once" do
+    expect(Hatchet::App.default_buildpack.object_id).to eq(Hatchet::App.default_buildpack.object_id)
+  end
+
+  it "nested deploy block only calls teardown once" do
+    @deploy = 0
+    app = Hatchet::App.new("default_ruby", buildpacks: [:default])
+    def app.in_dir_setup!; ;end # Don't create an app
+    def app.push_with_retry!; end # Don't try pushing to it
+    def app.teardown!; @teardown ||=0; @teardown += 1 end
+    def app.get_teardown_count; @teardown; end
+
+    app.deploy do |app|
+      @deploy += 1
+      app.deploy do |app|
+        @deploy += 1
+      end
+      app.deploy do |app|
+        @deploy += 1
+      end
+    end
+
+    expect(app.get_teardown_count).to eq(1)
+    expect(@deploy).to eq(3)
+  end
+end

data/spec/unit/reaper_spec.rb

@@ -18,7 +18,7 @@ describe "Reaper" do
   end
 
   describe "cycle" do
-    it "does not delete anything if under the limit" do
+    it "does not delete anything if no old apps" do
       reaper = Hatchet::Reaper.new(api_rate_limit: Object.new, hatchet_app_limit: 1, io: StringIO.new)
 
       def reaper.get_heroku_apps
@@ -29,7 +29,7 @@ describe "Reaper" do
       def reaper.check_get_heroku_apps_called; @called_get_heroku_apps ; end
       def reaper.reap_once; raise "should not be called"; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
 
       expect(reaper.check_get_heroku_apps_called).to be_truthy
     end
@@ -46,7 +46,7 @@ describe "Reaper" do
       end
       def reaper.destroy_called_with; @reaper_destroy_called_with; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
 
       expect(reaper.destroy_called_with).to eq({"name" => "hatchet-t-foo", "id" => 1})
     end
@@ -54,10 +54,10 @@ describe "Reaper" do
     it "sleeps, refreshes app list, and tries again when an old app is not past TTL" do
       warning = StringIO.new
       reaper = Hatchet::Reaper.new(
+        io: warning,
+        initial_sleep: 0,
         api_rate_limit: Object.new,
         hatchet_app_limit: 0,
-        initial_sleep: 0,
-        io: warning
       )
 
       def reaper.get_heroku_apps
@@ -74,7 +74,8 @@ describe "Reaper" do
 
       def reaper.get_slept_for_val; @_slept_for; end
 
-      reaper.clean_old_or_sleep
+      reaper.destroy_older_apps
+      reaper.sleep_if_over_limit(reason: "test")
 
       expect(reaper.get_slept_for_val).to eq(0)
       expect(reaper.destroy_called_with).to eq(nil)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heroku_hatchet
 version: !ruby/object:Gem::Version
-  version: 8.0.0
+  version: 8.0.2
 platform: ruby
 authors:
 - Richard Schneeman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-27 00:00:00.000000000 Z
+date: 2023-03-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: platform-api
@@ -160,6 +160,7 @@ extra_rdoc_files: []
 files:
 - ".github/workflows/check_changelog.yml"
 - ".github/workflows/ci.yml"
+- ".github/workflows/hatchet_app_cleaner.yml"
 - ".gitignore"
 - CHANGELOG.md
 - Gemfile
@@ -206,6 +207,7 @@ files:
 - spec/hatchet/local_repo_spec.rb
 - spec/hatchet/lock_spec.rb
 - spec/spec_helper.rb
+- spec/unit/app_spec.rb
 - spec/unit/default_ci_branch_spec.rb
 - spec/unit/heroku_run_spec.rb
 - spec/unit/init_spec.rb
@@ -246,6 +248,7 @@ test_files:
 - spec/hatchet/local_repo_spec.rb
 - spec/hatchet/lock_spec.rb
 - spec/spec_helper.rb
+- spec/unit/app_spec.rb
 - spec/unit/default_ci_branch_spec.rb
 - spec/unit/heroku_run_spec.rb
 - spec/unit/init_spec.rb