heroku_hatchet 8.0.0 → 8.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9e66d5aea4901cc03c37bbf6d0f4b567e49f9c170d471ff461cc642956a195e9
4
- data.tar.gz: 79fb324886b6a90828a753a9e93679db0386d1696f081efc33b05fef8d2fd5ab
3
+ metadata.gz: c72d006f643f893a0347e7c7ce5fae2e128d4a94fea501d957afb6738cc9374f
4
+ data.tar.gz: a507cd401042cf6e245b63824b9cbb30b79aa56541bc604f2c92b54d23413cbd
5
5
  SHA512:
6
- metadata.gz: d337d285c3c35fc705e2f715bf209df425a1d36e908cba200610123b41eab488a59439109634c2456c148d6f877d93e9910becfde2606582803853bd9bce5913
7
- data.tar.gz: 9c601a90d90d82782fd4fcdfca05ca1030738a1a0e93bd73c0cc42c3f60b91b7094b49b9128a3f4358ad4d161aceb2206850f90f155f2a1973529e705a00e345
6
+ metadata.gz: 6ba49829178ac76fe6a62faf32afbef57a828dc4fd390d50887f2b2c50f30a8359ad6f9639ad34b258a2594a885333e8a704b59f93e3bdf80cb87ce897f5a4c1
7
+ data.tar.gz: a54f637129f9b57f296f405e374082573b584b3c960bab387723f19f87bf8ee63562022a94ee08aa9e75bf3badffd63fdeec805897b71db30e15b74aaf5384c6
@@ -0,0 +1,31 @@
1
+ name: Hatchet app cleaner
2
+
3
+ on:
4
+ schedule:
5
+ # Daily at 6am UTC.
6
+ - cron: "0 6 * * *"
7
+ # Allow the workflow to be manually triggered too.
8
+ workflow_dispatch:
9
+
10
+ permissions:
11
+ contents: read
12
+
13
+ jobs:
14
+ hatchet-app-cleaner:
15
+ runs-on: ubuntu-latest
16
+ env:
17
+ HEROKU_API_KEY: ${{ secrets.HEROKU_API_KEY }}
18
+ HEROKU_API_USER: ${{ secrets.HEROKU_API_USER }}
19
+ HEROKU_DISABLE_AUTOUPDATE: 1
20
+ steps:
21
+ - name: Checkout
22
+ uses: actions/checkout@v3
23
+ - name: Install Ruby and dependencies
24
+ uses: ruby/setup-ruby@v1
25
+ with:
26
+ bundler-cache: true
27
+ ruby-version: "3.1"
28
+ - name: Run Hatchet destroy
29
+ # Only apps older than 10 minutes are destroyed, to ensure that any
30
+ # in progress CI runs are not interrupted.
31
+ run: bundle exec hatchet destroy --older-than 10
data/CHANGELOG.md CHANGED
@@ -1,5 +1,13 @@
1
1
  ## HEAD
2
2
 
3
+ ## 8.0.2
4
+
5
+ - Bugfix: Allow nested deploy blocks with new teardown logic (https://github.com/heroku/hatchet/pull/201)
6
+
7
+ ## 8.0.1
8
+
9
+ - Bugfix: Lock and sleep and refresh API when duplicate app deletion detected (https://github.com/heroku/hatchet/pull/198)
10
+
3
11
  ## 8.0.0
4
12
 
5
13
  - Breaking change: Delete apps on teardown. Previously hatchet would delete apps lazily to help with debugging. This behavior allowed developers to inspect logs and `heroku run bash` in the event of an unexpected failure. In practice, it is rarely needed and causes accounts to retain apps indefinitely. Previously there was no cost to retaining applications, but now `basic` applications incur a charge. Change details:
data/bin/hatchet CHANGED
@@ -123,7 +123,11 @@ class HatchetCLI < Thor
123
123
  when options[:older_than]
124
124
  minutes = options[:older_than].to_i
125
125
  puts "Destroying apps older than #{minutes}m"
126
- reaper.destroy_older_apps(minutes: minutes)
126
+ reaper.destroy_older_apps(
127
+ minutes: minutes,
128
+ force_refresh: true,
129
+ on_conflict: :refresh_api_and_continue
130
+ )
127
131
  puts "Done"
128
132
  else
129
133
  raise "No flags given run `hatchet help destroy` for options"
data/lib/hatchet/app.rb CHANGED
@@ -77,6 +77,7 @@ module Hatchet
77
77
  @buildpacks.map! {|b| b == :default ? self.class.default_buildpack : b}
78
78
  @run_multi = run_multi
79
79
  @max_retries_count = retries
80
+ @outer_deploy_block = nil
80
81
 
81
82
  if run_multi && !ENV["HATCHET_EXPENSIVE_MODE"]
82
83
  raise "You're attempting to enable `run_multi: true` mode, but have not enabled `HATCHET_EXPENSIVE_MODE=1` env var to verify you understand the risks"
@@ -280,14 +281,31 @@ module Hatchet
280
281
  def create_app
281
282
  3.times.retry do
282
283
  begin
283
- @reaper.destroy_older_apps
284
+ # Remove any obviously old apps first
285
+ # Try to use existing cache of apps to
286
+ # minimize API calls
287
+ @reaper.destroy_older_apps(
288
+ force_refresh: false,
289
+ on_conflict: :stop_if_under_limit,
290
+ )
284
291
  hash = { name: name, stack: stack }
285
292
  hash.delete_if { |k,v| v.nil? }
286
293
  result = heroku_api_create_app(hash)
287
294
  @heroku_id = result["id"]
288
295
  rescue => e
289
- puts "Warning: Could not create app #{e.message}"
290
- @reaper.clean_old_or_sleep
296
+ # If we can't create an app assume
297
+ # it might be due to resource constraints
298
+ #
299
+ # Try to delete existing apps
300
+ @reaper.destroy_older_apps(
301
+ force_refresh: true,
302
+ on_conflict: :stop_if_under_limit,
303
+ )
304
+ # If we're still not under the limit, sleep a bit
305
+ # retry later.
306
+ @reaper.sleep_if_over_limit(
307
+ reason: "Could not create app #{e.message}"
308
+ )
291
309
  raise e
292
310
  end
293
311
  end
@@ -407,13 +425,14 @@ module Hatchet
407
425
  def deploy(&block)
408
426
  in_directory do
409
427
  annotate_failures do
428
+ @outer_deploy_block ||= block # deploy! can be called multiple times. Only teardown once
410
429
  in_dir_setup!
411
- self.push_with_retry!
430
+ push_with_retry!
412
431
  block.call(self, api_rate_limit.call, output) if block_given?
413
432
  end
414
433
  end
415
434
  ensure
416
- self.teardown! if block_given?
435
+ self.teardown! if block_given? && @outer_deploy_block == block
417
436
  end
418
437
 
419
438
  def push
@@ -28,6 +28,13 @@ module Hatchet
28
28
  DEFAULT_REGEX = /^#{Regexp.escape(Hatchet::APP_PREFIX)}[a-f0-9]+/
29
29
  TTL_MINUTES = ENV.fetch("HATCHET_ALIVE_TTL_MINUTES", "7").to_i
30
30
 
31
+ # Protect against parallel deletion on the same machine
32
+ # via concurrent processes
33
+ #
34
+ # Does not protect against distributed systems on different
35
+ # machines trying to delete the same applications
36
+ MUTEX_FILE = File.open(File.join(Dir.tmpdir(), "hatchet_reaper_mutex"), File::CREAT)
37
+
31
38
  attr_accessor :io, :hatchet_app_limit
32
39
 
33
40
  def initialize(api_rate_limit: , regex: DEFAULT_REGEX, io: STDOUT, hatchet_app_limit: HATCHET_APP_LIMIT, initial_sleep: 10)
@@ -39,60 +46,127 @@ module Hatchet
39
46
  @reaper_throttle = ReaperThrottle.new(initial_sleep: initial_sleep)
40
47
  end
41
48
 
42
- # Called when we need an app, but are over limit or
43
- # if an exception has occured that was possibly triggered
44
- # by apps being over limit
45
- def clean_old_or_sleep
46
- # Protect against parallel deletion of the same app on the same system
47
- mutex_file = File.open("#{Dir.tmpdir()}/hatchet_reaper_mutex", File::CREAT)
48
- mutex_file.flock(File::LOCK_EX)
49
-
50
- destroy_older_apps(force_refresh: true)
51
-
52
- if @apps.length > @limit
49
+ def sleep_if_over_limit(reason: )
50
+ if @apps.length >= @limit
53
51
  age = AppAge.new(created_at: @apps.last["created_at"], ttl_minutes: TTL_MINUTES)
54
52
  @reaper_throttle.call(max_sleep: age.sleep_for_ttl) do |sleep_for|
55
53
  io.puts <<-EOM.strip_heredoc
56
54
  WARNING: Hatchet app limit reached (#{@apps.length}/#{@limit})
57
- All known apps are younger than #{TTL_MINUTES} minutes
55
+ All known apps are younger than #{TTL_MINUTES} minutes.
56
+ Sleeping (#{sleep_for}s)
57
+
58
+ Reason: #{reason}
58
59
  EOM
59
60
 
60
61
  sleep(sleep_for)
61
62
  end
62
63
  end
63
- ensure
64
- mutex_file.close
65
64
  end
66
65
 
67
66
  # Destroys apps that are older than the given argument (expecting integer minutes)
68
- def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?)
67
+ #
68
+ # This method might be running concurrently on multiple processes or multiple
69
+ # machines.
70
+ #
71
+ # When a duplicate destroy is detected we can move forward with a conflict strategy:
72
+ #
73
+ # - `:refresh_api_and_continue`: Sleep to see if another process will clean up everything for
74
+ # us and then re-populate apps from the API and continue.
75
+ # - `:stop_if_under_limit`: Sleep to allow other processes to continue. Then if apps list
76
+ # is under the limit, assume someone else is already cleaning up for us and that we're
77
+ # good to move ahead to try to create an app. Otherwise if we're at or
78
+ # over the limit sleep, refresh the app list, and continue attempting to delete apps.
79
+ def destroy_older_apps(minutes: TTL_MINUTES, force_refresh: @apps.empty?, on_conflict: :refresh_api_and_continue)
80
+ MUTEX_FILE.flock(File::LOCK_EX)
81
+
69
82
  refresh_app_list if force_refresh
70
83
 
71
- @apps.each do |app|
84
+ while app = @apps.pop
72
85
  age = AppAge.new(created_at: app["created_at"], ttl_minutes: minutes)
73
- if age.can_delete?
74
- destroy_with_log(
75
- name: app["name"],
76
- id: app["id"],
77
- reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
78
- )
86
+ if !age.can_delete?
87
+ @apps.push(app)
88
+ break
89
+ else
90
+ begin
91
+ destroy_with_log(
92
+ id: app["id"],
93
+ name: app["name"],
94
+ reason: "app age (#{age.in_minutes}m) is older than #{minutes}m"
95
+ )
96
+ rescue AlreadyDeletedError => e
97
+ if handle_conflict(
98
+ strategy: on_conflict,
99
+ conflict_message: e.message,
100
+ ) == :stop
101
+ break
102
+ end
103
+ end
79
104
  end
80
- rescue AlreadyDeletedError
81
- # Ignore, keep going
82
105
  end
106
+ ensure
107
+ MUTEX_FILE.flock(File::LOCK_UN)
83
108
  end
84
109
 
85
110
  # No guardrails, will delete all apps that match the hatchet namespace
86
111
  def destroy_all(force_refresh: @apps.empty?)
112
+ MUTEX_FILE.flock(File::LOCK_EX)
113
+
87
114
  refresh_app_list if force_refresh
88
115
 
89
- @apps.each do |app|
116
+ while app = @apps.pop
90
117
  begin
91
118
  destroy_with_log(name: app["name"], id: app["id"], reason: "destroy all")
92
- rescue AlreadyDeletedError
93
- # Ignore, keep going
119
+ rescue AlreadyDeletedError => e
120
+ handle_conflict(
121
+ conflict_message: e.message,
122
+ strategy: :refresh_api_and_continue
123
+ )
94
124
  end
95
125
  end
126
+ ensure
127
+ MUTEX_FILE.flock(File::LOCK_UN)
128
+ end
129
+
130
+ # Will sleep with backoff and emit a warning message
131
+ # returns :continue or :stop symbols
132
+ # :stop indicates execution should stop
133
+ private def handle_conflict(conflict_message:, strategy:)
134
+ message = String.new(<<-EOM.strip_heredoc)
135
+ WARNING: Possible race condition detected: #{conflict_message}
136
+ Hatchet app limit (#{@apps.length}/#{@limit}), using strategy #{strategy}
137
+ EOM
138
+
139
+ conflict_state = if :refresh_api_and_continue == strategy
140
+ message << "\nSleeping, refreshing app list, and continuing."
141
+ :continue
142
+ elsif :stop_if_under_limit == strategy && @apps.length >= @limit
143
+ message << "\nSleeping, refreshing app list, and continuing. Not under limit."
144
+ :continue
145
+ elsif :stop_if_under_limit == strategy
146
+ message << "\nHalting deletion of older apps. Under limit."
147
+ :stop
148
+ else
149
+ raise "No such strategy: #{strategy}, plese use :stop_if_under_limit or :refresh_api_and_continue"
150
+ end
151
+
152
+ @reaper_throttle.call(max_sleep: TTL_MINUTES) do |sleep_for|
153
+ io.puts <<-EOM.strip_heredoc
154
+ #{message}
155
+ Sleeping (#{sleep_for}s)
156
+ EOM
157
+
158
+ sleep(sleep_for)
159
+ end
160
+
161
+ case conflict_state
162
+ when :continue
163
+ refresh_app_list
164
+ when :stop
165
+ else
166
+ raise "Unknown state #{conflict_state}"
167
+ end
168
+
169
+ conflict_state
96
170
  end
97
171
 
98
172
  private def get_heroku_apps
@@ -117,15 +191,15 @@ module Hatchet
117
191
  body = e.response.body
118
192
  request_id = e.response.headers["Request-Id"]
119
193
  if body =~ /Couldn\'t find that app./
120
- io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
121
- raise AlreadyDeletedError.new
194
+ message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 404, request_id: #{request_id}"
195
+ raise AlreadyDeletedError.new(message)
122
196
  else
123
197
  raise e
124
198
  end
125
199
  rescue Excon::Error::Forbidden => e
126
200
  request_id = e.response.headers["Request-Id"]
127
- io.puts "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
128
- raise AlreadyDeletedError.new
201
+ message = "Duplicate destroy attempted #{name.inspect}: #{id}, status: 403, request_id: #{request_id}"
202
+ raise AlreadyDeletedError.new(message)
129
203
  end
130
204
  end
131
205
  end
@@ -1,3 +1,3 @@
1
1
  module Hatchet
2
- VERSION = "8.0.0"
2
+ VERSION = "8.0.2"
3
3
  end
@@ -1,20 +1,6 @@
1
1
  require("spec_helper")
2
2
 
3
3
  describe "AppTest" do
4
- it "annotates rspec expectation failures" do
5
- app = Hatchet::Runner.new("default_ruby")
6
- error = nil
7
- begin
8
- app.annotate_failures do
9
- expect(true).to eq(false)
10
- end
11
- rescue RSpec::Expectations::ExpectationNotMetError => e
12
- error = e
13
- end
14
-
15
- expect(error.message).to include(app.name)
16
- end
17
-
18
4
  it "does not modify local files by mistake" do
19
5
  Dir.mktmpdir do |dir_1|
20
6
  app = Hatchet::Runner.new(dir_1)
@@ -29,7 +15,6 @@ describe "AppTest" do
29
15
  entries_array -= ["..", ".", "foo.txt"]
30
16
  expect(entries_array).to be_empty
31
17
 
32
-
33
18
  entries_array = Dir.entries(dir_1)
34
19
  entries_array -= ["..", ".", "foo.txt"]
35
20
  expect(entries_array).to be_empty
@@ -37,33 +22,8 @@ describe "AppTest" do
37
22
  end
38
23
  end
39
24
 
40
- it "calls reaper if cannot create an app" do
41
- app = Hatchet::App.new("default_ruby", buildpacks: [:default])
42
- def app.heroku_api_create_app(*args); raise StandardError.new("made you look"); end
43
-
44
- reaper = app.reaper
45
-
46
- def reaper.clean_old_or_sleep; @app_exception_message = true; end
47
- def reaper.clean_old_was_called?; @app_exception_message; end
48
-
49
- expect {
50
- app.create_app
51
- }.to raise_error("made you look")
52
-
53
- expect(reaper.clean_old_was_called?).to be_truthy
54
- end
55
-
56
- it "app with default" do
57
- app = Hatchet::App.new("default_ruby", buildpacks: [:default])
58
- expect(app.buildpacks.first).to match("https://github.com/heroku/heroku-buildpack-ruby")
59
- end
60
-
61
- it "default_buildpack is only computed once" do
62
- expect(Hatchet::App.default_buildpack.object_id).to eq(Hatchet::App.default_buildpack.object_id)
63
- end
64
-
65
25
  it "create app with stack" do
66
- stack = "heroku-18"
26
+ stack = "heroku-20"
67
27
  app = Hatchet::App.new("default_ruby", stack: stack)
68
28
  app.create_app
69
29
  expect(app.platform_api.app.info(app.name)["build_stack"]["name"]).to eq(stack)
@@ -0,0 +1,65 @@
1
+ require "spec_helper"
2
+
3
+ # Tests in this file do not deploy to Heroku
4
+ describe "App unit tests" do
5
+ it "annotates rspec expectation failures" do
6
+ app = Hatchet::Runner.new("default_ruby")
7
+ error = nil
8
+ begin
9
+ app.annotate_failures do
10
+ expect(true).to eq(false)
11
+ end
12
+ rescue RSpec::Expectations::ExpectationNotMetError => e
13
+ error = e
14
+ end
15
+
16
+ expect(error.message).to include(app.name)
17
+ end
18
+
19
+ it "calls reaper if cannot create an app" do
20
+ app = Hatchet::App.new("default_ruby", buildpacks: [:default])
21
+ def app.heroku_api_create_app(*args); raise StandardError.new("made you look"); end
22
+
23
+ reaper = app.reaper
24
+
25
+ def reaper.destroy_older_apps(*args, **kwargs, &block); @app_exception_message = true; end
26
+ def reaper.clean_old_was_called?; @app_exception_message; end
27
+
28
+ expect {
29
+ app.create_app
30
+ }.to raise_error("made you look")
31
+
32
+ expect(reaper.clean_old_was_called?).to be_truthy
33
+ end
34
+
35
+ it "app with default" do
36
+ app = Hatchet::App.new("default_ruby", buildpacks: [:default])
37
+ expect(app.buildpacks.first).to match("https://github.com/heroku/heroku-buildpack-ruby")
38
+ end
39
+
40
+ it "default_buildpack is only computed once" do
41
+ expect(Hatchet::App.default_buildpack.object_id).to eq(Hatchet::App.default_buildpack.object_id)
42
+ end
43
+
44
+ it "nested deploy block only calls teardown once" do
45
+ @deploy = 0
46
+ app = Hatchet::App.new("default_ruby", buildpacks: [:default])
47
+ def app.in_dir_setup!; ;end # Don't create an app
48
+ def app.push_with_retry!; end # Don't try pushing to it
49
+ def app.teardown!; @teardown ||=0; @teardown += 1 end
50
+ def app.get_teardown_count; @teardown; end
51
+
52
+ app.deploy do |app|
53
+ @deploy += 1
54
+ app.deploy do |app|
55
+ @deploy += 1
56
+ end
57
+ app.deploy do |app|
58
+ @deploy += 1
59
+ end
60
+ end
61
+
62
+ expect(app.get_teardown_count).to eq(1)
63
+ expect(@deploy).to eq(3)
64
+ end
65
+ end
@@ -18,7 +18,7 @@ describe "Reaper" do
18
18
  end
19
19
 
20
20
  describe "cycle" do
21
- it "does not delete anything if under the limit" do
21
+ it "does not delete anything if no old apps" do
22
22
  reaper = Hatchet::Reaper.new(api_rate_limit: Object.new, hatchet_app_limit: 1, io: StringIO.new)
23
23
 
24
24
  def reaper.get_heroku_apps
@@ -29,7 +29,7 @@ describe "Reaper" do
29
29
  def reaper.check_get_heroku_apps_called; @called_get_heroku_apps ; end
30
30
  def reaper.reap_once; raise "should not be called"; end
31
31
 
32
- reaper.clean_old_or_sleep
32
+ reaper.destroy_older_apps
33
33
 
34
34
  expect(reaper.check_get_heroku_apps_called).to be_truthy
35
35
  end
@@ -46,7 +46,7 @@ describe "Reaper" do
46
46
  end
47
47
  def reaper.destroy_called_with; @reaper_destroy_called_with; end
48
48
 
49
- reaper.clean_old_or_sleep
49
+ reaper.destroy_older_apps
50
50
 
51
51
  expect(reaper.destroy_called_with).to eq({"name" => "hatchet-t-foo", "id" => 1})
52
52
  end
@@ -54,10 +54,10 @@ describe "Reaper" do
54
54
  it "sleeps, refreshes app list, and tries again when an old app is not past TTL" do
55
55
  warning = StringIO.new
56
56
  reaper = Hatchet::Reaper.new(
57
+ io: warning,
58
+ initial_sleep: 0,
57
59
  api_rate_limit: Object.new,
58
60
  hatchet_app_limit: 0,
59
- initial_sleep: 0,
60
- io: warning
61
61
  )
62
62
 
63
63
  def reaper.get_heroku_apps
@@ -74,7 +74,8 @@ describe "Reaper" do
74
74
 
75
75
  def reaper.get_slept_for_val; @_slept_for; end
76
76
 
77
- reaper.clean_old_or_sleep
77
+ reaper.destroy_older_apps
78
+ reaper.sleep_if_over_limit(reason: "test")
78
79
 
79
80
  expect(reaper.get_slept_for_val).to eq(0)
80
81
  expect(reaper.destroy_called_with).to eq(nil)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: heroku_hatchet
3
3
  version: !ruby/object:Gem::Version
4
- version: 8.0.0
4
+ version: 8.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Richard Schneeman
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-27 00:00:00.000000000 Z
11
+ date: 2023-03-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: platform-api
@@ -160,6 +160,7 @@ extra_rdoc_files: []
160
160
  files:
161
161
  - ".github/workflows/check_changelog.yml"
162
162
  - ".github/workflows/ci.yml"
163
+ - ".github/workflows/hatchet_app_cleaner.yml"
163
164
  - ".gitignore"
164
165
  - CHANGELOG.md
165
166
  - Gemfile
@@ -206,6 +207,7 @@ files:
206
207
  - spec/hatchet/local_repo_spec.rb
207
208
  - spec/hatchet/lock_spec.rb
208
209
  - spec/spec_helper.rb
210
+ - spec/unit/app_spec.rb
209
211
  - spec/unit/default_ci_branch_spec.rb
210
212
  - spec/unit/heroku_run_spec.rb
211
213
  - spec/unit/init_spec.rb
@@ -246,6 +248,7 @@ test_files:
246
248
  - spec/hatchet/local_repo_spec.rb
247
249
  - spec/hatchet/lock_spec.rb
248
250
  - spec/spec_helper.rb
251
+ - spec/unit/app_spec.rb
249
252
  - spec/unit/default_ci_branch_spec.rb
250
253
  - spec/unit/heroku_run_spec.rb
251
254
  - spec/unit/init_spec.rb