heroku_hatchet 7.2.0 → 7.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2dd4a304828fca43576fed5a71b7be305142c3220c7dd020bff44b17a4a070a0
-  data.tar.gz: c0dee0a8ffb794f10a3dbb96beeafa26dd5f95b489eb9638a40bf408dce44c11
+  metadata.gz: a45ee1ab2021838074b4f288d70aab769a71961cbae7cd12af67866e5558bb64
+  data.tar.gz: 0c99b826bfc797632924e151b796590f1ddbd07ff50e9eea2dc816b9991fb69d
 SHA512:
-  metadata.gz: f6fa0cef6167e9b4985b08f01e42f33410632d4dfb083551d2641e7009745a079044f606cc4456ed8b8f0dfb0ec087b9b3197d1dabb9a0c41cd7f96ee978e22c
-  data.tar.gz: 50689c87372d4df2fc7d354ccc0f6f870285283869a190f3272f76d0d238e2d3cd13ca485ca6436916a6976c847fbd2be057fbe4e99ed1aa8ac97abdfef9eeb9
+  metadata.gz: 940afbe59938c35c5bee92ac43dabe66e0e68f7cefbc35b662667eaecec343f04cc83e93418e7302a47ce2f955b2d7269073e167614a475dc91d69e6bee94d82
+  data.tar.gz: 204098d3744d64717141ca4d8f754324e761981b3d2d3bc9710e3b18e5532c97a1028813e1d7e2d295d35fb36e1b09ab78761a370af6490849de991ca50045ce

data/.circleci/config.yml

@@ -29,9 +29,9 @@ references:
         bundle clean
 
 jobs:
-  "ruby-2.5":
+  "ruby-2.2":
     docker:
-      - image: circleci/ruby:2.5
+      - image: circleci/ruby:2.2
     steps:
       - checkout
       - <<: *restore
@@ -39,9 +39,9 @@ jobs:
       - <<: *hatchet_setup
       - <<: *unit
       - <<: *save
-  "ruby-2.6":
+  "ruby-2.5":
     docker:
-      - image: circleci/ruby:2.6
+      - image: circleci/ruby:2.5
     steps:
       - checkout
       - <<: *restore
@@ -64,6 +64,6 @@ workflows:
   version: 2
   build:
     jobs:
+      - "ruby-2.2"
       - "ruby-2.5"
-      - "ruby-2.6"
       - "ruby-2.7"

data/CHANGELOG.md

@@ -1,6 +1,51 @@
 ## HEAD
 
-## 7.1.4
+## 7.3.4
+
+- Memoize `Hatchet::App.default_buildpack` (https://github.com/heroku/hatchet/pull/183)
+- Fix repository path lookup when custom Hatchet directory set (https://github.com/heroku/hatchet/issues/181)
+- Handle additional variations of rate limit error messages (https://github.com/heroku/hatchet/pull/182)
+- Add `HATCHET_DEFAULT_STACK` for configuring the default stack (https://github.com/heroku/hatchet/pull/184)
+- Fix typo in the reaper `"Duplicate destroy attempted"` message (https://github.com/heroku/hatchet/pull/175)
+- Set `init.defaultBranch` in ci:setup to suppress `git init` warning in Git 2.30+ (https://github.com/heroku/hatchet/issues/172)
+- Switch `heroku ci:install_heroku` to the Heroku CLI standalone installer rather than the APT install method (https://github.com/heroku/hatchet/issues/171)
+
+## 7.3.3
+
+- Quiet personal tokens (https://github.com/heroku/hatchet/pull/148)
+
+## 7.3.2
+
+- Fix App#in_directory_fork not receiving debugging output when an error is raised (https://github.com/heroku/hatchet/pull/146)
+- Do not create CI tarball inside cwd to prevent tar "file changed as we read it" warnings.
+
+## 7.3.1
+
+- Fix Ruby incompatibility introduced by using `&.` and `rescue` without
+  `begin`/`end` without a `required_ruby_version` in hatchet.gemspec.
+  (https://github.com/heroku/hatchet/pull/139)
+
+## 7.3.0
+
+- Deprecations
+  - Deprecation: Calling `App#before_deploy` as a way to clear/replace the existing block should now be done with `App#before_deploy(:replace)` (https://github.com/heroku/hatchet/pull/126)
+  - Deprecation: HATCHET_BUILDPACK_BASE default (https://github.com/heroku/hatchet/pull/133)
+  - Deprecation: App#directory (https://github.com/heroku/hatchet/pull/135)
+
+- Flappy test improvements
+  - Increase CI timeout limit to 900 seconds (15 minutes) (https://github.com/heroku/hatchet/pull/137)
+  - Empty string returns from App#run now trigger retries (https://github.com/heroku/hatchet/pull/132)
+  - Rescue 403 on pipeline delete (https://github.com/heroku/hatchet/pull/130)
+  - Additional rate throttle cases handled (https://github.com/heroku/hatchet/pull/128)
+
+- Usability
+  - Annotate rspec expectation failures inside of deploy blocks with hatchet debug information (https://github.com/heroku/hatchet/pull/136)
+  - Hatchet#new raises a helpful error when no source code location is provided (https://github.com/heroku/hatchet/pull/134)
+  - Lazy evaluation of HATCHET_BUILDPACK_BASE env var (https://github.com/heroku/hatchet/pull/133)
+  - Allow multiple `App#before_deploy` blocks to be set and called (https://github.com/heroku/hatchet/pull/126)
+  - Performance improvement when running without an explicit HEROKU_API_KEY set (https://github.com/heroku/hatchet/pull/128)
+
+## 7.2.0
 
 - App#setup! no longer modifies files on disk. (https://github.com/heroku/hatchet/pull/125)
 - Add `$ hatchet init` command for bootstrapping new projects (https://github.com/heroku/hatchet/pull/123)
@@ -309,7 +354,6 @@
 
     after_script: bundle exec rake hatchet:teardown_travis
 
-
 ## 0.1.1
 
 - Allow auto retries of pushes by setting environment variable `HATCHET_RETRIES=3`

data/README.md

@@ -504,7 +504,7 @@ The `Hatchet::Runner.new` takes several arguments.
 
 ### Init options
 
-- stack (String): The stack you want to deploy to on Heroku.
+- stack (String): The Heroku [stack](https://devcenter.heroku.com/articles/stack) to use for the app. If this is not set, the stack will be determined from `HATCHET_DEFAULT_STACK`, or else the Heroku platform's [default stack](https://devcenter.heroku.com/articles/stack#default-stack).
 
 ```ruby
 Hatchet::Runner.new("default_ruby", stack: "heroku-16").deploy do |app|
@@ -525,7 +525,7 @@ end
 
 In this example, the app would use the nodejs buildpack, and then `:default` gets replaced by your Git url and branch name.
 
-- before_deploy (Block): Instead of using the `tap` syntax you can provide a block directly to hatchet app initialization:
+- before_deploy (Block): Instead of using the `tap` syntax you can provide a block directly to hatchet app initialization. Example:
 
 ```ruby
 Hatchet::Runner.new("default_ruby", before_deploy: ->{ FileUtils.touch("foo.txt")}).deploy do
@@ -598,6 +598,7 @@ app.get_config("DEPLOY_TASKS") # => "run:bloop"
 
 - `app.set_lab()`: Enables the specified lab/feature on the app
 - `app.add_database()`: adds a database to the app, defaults to the "dev" database
+- `app.update_stack()`: Change the app's stack to that specified (for example `"heroku-20"`). Will take effect on the next build.
 - `app.run()`: Runs a `heroku run bash` session with the arguments, covered above.
 - `app.run_multi()`: Runs a `heroku run bash` session in the background and yields the results. This requires the `run_multi` flag of the app to be set to `true`, which will charge your application (the `HATCHET_EXPENSIVE_MODE` env var must also be set to use this feature). Example above.
 - `app.create_app`: Can be used to manually create the app without deploying it (You probably want `setup!` though)
@@ -630,6 +631,23 @@ Hatchet::Runner.new("default_ruby", before_deploy: before_deploy_proc).deploy do
 end
 ```
 
+You can call multiple blocks by specifying (`:prepend` or `:append`):
+
+```ruby
+Hatchet::Runner.new("default_ruby").tap do |app|
+  app.before_deploy do
+    FileUtils.touch("foo.txt")
+  end
+
+  app.before_deploy(:append) do
+    FileUtils.touch("bar.txt")
+  end
+  app.deploy do
+  end
+end
+```
+
+
 - `app.commit!`: Will updates the contents of your local git dir if you've modified files on disk
 
 ```ruby
@@ -660,7 +678,7 @@ end
 > Note: If you want to execute tests in this temp directory, you likely want to use `in_directory_fork` otherwise, you might accidentally contaminate the current environment's variables if you modify them.
 
 - `app.in_directory_fork`: Runs the given block in a temp directory and inside of a forked process, an example given above.
-- `app.directory`: Returns the directory of the example application on disk, this is NOT the temp directory that you're currently executing against. It's probably not what you want.
+- `app.original_source_code_directory`: Returns the directory of the example application on disk, this is NOT the temp directory that you're currently executing against. It's probably not what you want.
 - `app.deploy`: Your main method takes a block to execute after the deploy is successful. If no block is provided, you must manually call `app.teardown!` (see below for an example).
 - `app.output`: The output contents of the deploy
 - `app.platform_api`: Returns an instance of the [platform-api Heroku client](https://github.com/heroku/platform-api). If Hatchet doesn't give you access to a part of Heroku that you need, you can likely do it with the platform-api client.
@@ -710,6 +728,7 @@ HATCHET_ALIVE_TTL_MINUTES=7
 - `HATCHET_APP_LIMIT`: The maximum number of **hatchet** apps that Hatchet will allow in the given account before running the reaper. For local execution, keep this low as you don't want your account dominated by hatchet apps. For CI, you want it to be much larger, 80-100 since it's not competing with non-hatchet apps. Your test runner account needs to be a dedicated account.
 - `HEROKU_API_KEY`: The API key of your test account user. If you run locally without this set, it will use your personal credentials.
 - `HEROKU_API_USER`: The email address of your user account. If you run locally without this set, it will use your personal credentials.
+- `HATCHET_DEFAULT_STACK`: The default Heroku stack to be used when an explicit `stack` is not passed to `App.new`. If this is not set, apps will instead use Heroku platform's [default stack](https://devcenter.heroku.com/articles/stack#default-stack).
 - `HATCHET_RUN_MULTI`: If enabled, this will scale up deployed apps to "standard-1x" once deployed instead of running on the free tier. This enables the `run_multi` method capability, however scaling up is not free. WARNING: Setting this env var will incur charges to your Heroku account. We recommended never to enable this setting unless you work for Heroku. To use this you must also set `HATCHET_EXPENSIVE_MODE=1`
 - `HATCHET_EXPENSIVE_MODE`: This is intended to be a "safety" environment variable. If it is not set, Hatchet will prevent you from using the `run_multi: true` setting or the `HATCHET_RUN_MULTI` environment variables. There are still ways to incur charges without this feature, but unless you're absolutely confident your test setup will not leave "orphan" apps that are billing you, do not enable this setting. Even then, only set this value if you work for Heroku. To recap WARNING: setting this is expensive.
 

data/bin/hatchet

@@ -52,7 +52,7 @@ class HatchetCLI < Thor
       Threaded.later do
         commit = lock_hash[directory]
         directory = File.expand_path(directory)
-        if !Dir[directory]&.empty?
+        if !(Dir[directory] && Dir[directory].empty?)
           puts "== pulling '#{git_repo}' into '#{directory}'\n"
           pull(directory, git_repo)
         else

data/etc/ci_setup.rb

@@ -30,6 +30,8 @@ run_cmd "bundle exec hatchet ci:install_heroku"
 run_cmd "bundle exec hatchet install"
 run_cmd "git config --get user.email > /dev/null || git config --global user.email #{ENV.fetch('HEROKU_API_USER').shellescape}"
 run_cmd "git config --get user.name > /dev/null || git config --global user.name 'BuildpackTester'"
+# Suppress the `git init` warning in Git 2.30+ when no default branch name is set.
+run_cmd "git config --global init.defaultBranch main"
 
 puts "== Done =="
 

data/etc/setup_heroku.sh

@@ -1,4 +1,4 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh
+curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install.sh | sh

data/hatchet.gemspec

@@ -17,17 +17,17 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+  gem.required_ruby_version = '>= 2.2.0'
 
   gem.add_dependency "platform-api",  "~> 3"
   gem.add_dependency "rrrretry",      "~> 1"
   gem.add_dependency "excon",         "~> 0"
-  gem.add_dependency "thor",          "~> 0"
+  gem.add_dependency "thor",          "~> 1"
   gem.add_dependency "threaded",      "~> 0"
 
   gem.add_development_dependency "rspec"
   gem.add_development_dependency "rake",           ">= 10"
   gem.add_development_dependency "mocha",          ">= 1"
   gem.add_development_dependency "parallel_split_test"
-  gem.add_development_dependency "travis",         ">= 1"
   gem.add_development_dependency "rspec-retry"
 end

data/lib/hatchet.rb

@@ -19,6 +19,7 @@ require 'hatchet/git_app'
 require 'hatchet/config'
 require 'hatchet/api_rate_limit'
 require 'hatchet/init_project'
+require 'hatchet/heroku_run'
 
 module Hatchet
   RETRIES = Integer(ENV['HATCHET_RETRIES']   || 1)
@@ -45,3 +46,22 @@ module Hatchet
     end
   end
 end
+
+unless ::String.instance_methods.include?(:strip_heredoc)
+  # We can get rid of this when all rubies can support <<~ syntax
+  class ::String
+    def strip_heredoc
+      gsub(/^#{scan(/^[ \t]*(?=\S)/).min}/, "".freeze)
+    end
+  end
+end
+
+unless ::String.instance_methods.include?(:match?)
+  # We can get rid of this when all rubies can support String#match? method
+  class ::String
+    def match?(value)
+      self =~ value
+    end
+  end
+end
+

data/lib/hatchet/app.rb

@@ -5,11 +5,15 @@ require 'tmpdir'
 
 module Hatchet
   class App
-    HATCHET_BUILDPACK_BASE   = (ENV['HATCHET_BUILDPACK_BASE'] || "https://github.com/heroku/heroku-buildpack-ruby.git")
+    HATCHET_BUILDPACK_BASE = -> {
+      ENV.fetch('HATCHET_BUILDPACK_BASE') {
+        warn "ENV HATCHET_BUILDPACK_BASE is not set. It currently defaults to the ruby buildpack. In the future this env var will be required"
+        "https://github.com/heroku/heroku-buildpack-ruby.git"
+      }
+    }
     HATCHET_BUILDPACK_BRANCH = -> { ENV['HATCHET_BUILDPACK_BRANCH'] || ENV['HEROKU_TEST_RUN_BRANCH'] || Hatchet.git_branch }
-    BUILDPACK_URL = "https://github.com/heroku/heroku-buildpack-ruby.git"
 
-    attr_reader :name, :stack, :directory, :repo_name, :app_config, :buildpacks, :reaper, :max_retries_count
+    attr_reader :name, :stack, :repo_name, :app_config, :buildpacks, :reaper, :max_retries_count
 
     class FailedDeploy < StandardError; end
 
@@ -18,7 +22,7 @@ module Hatchet
 
       def initialize(app, message, output: )
         @output = output
-        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.original_source_code_directory}'\n"
         msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
         msg << "#{message}\n"
         msg << "output:\n"
@@ -32,7 +36,7 @@ module Hatchet
 
       def initialize(app, message, output: )
         @output = output
-        msg = "Could not release '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg = "Could not release '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.original_source_code_directory}'\n"
         msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
         msg << "#{message}\n"
         msg << "output:\n"
@@ -42,9 +46,10 @@ module Hatchet
     end
 
     SkipDefaultOption = Object.new
+    DEFAULT_REPO_NAME = Object.new
 
-    def initialize(repo_name,
-                   stack: "",
+    def initialize(repo_name = DEFAULT_REPO_NAME,
+                   stack: ENV["HATCHET_DEFAULT_STACK"],
                    name: default_name,
                    debug: nil,
                    debugging: nil,
@@ -58,6 +63,7 @@ module Hatchet
                    retries: RETRIES,
                    config: {}
                   )
+      raise "You tried creating a Hatchet::App instance without source code, pass in a path to an app to deploy or the name of an app in your hatchet.json" if repo_name == DEFAULT_REPO_NAME
       @repo_name     = repo_name
       @directory     = self.config.path_for_name(@repo_name)
       @name          = name
@@ -78,13 +84,37 @@ module Hatchet
       @already_in_dir = nil
       @app_is_setup = nil
 
-      @before_deploy = before_deploy
+      @before_deploy_array = []
+      @before_deploy_array << before_deploy if before_deploy
       @app_config    = config
       @reaper        = Reaper.new(api_rate_limit: api_rate_limit)
     end
 
+    private def test_failure_classes
+      class_array = []
+      class_array << RSpec::Expectations::ExpectationNotMetError if defined?(RSpec::Expectations::ExpectationNotMetError)
+      class_array
+    end
+
+    def annotate_failures
+      yield
+    rescue *test_failure_classes => e
+      raise e, "App: #{name} (#{@repo_name})\n#{e.message}"
+    end
+
+    def directory
+      warn "Calling App#directory returns the original location of the app's source code that should not be modified, if this is really what you want use `original_source_code_directory` instead."
+      warn caller
+      @directory
+    end
+
+    def original_source_code_directory
+      @directory
+    end
+
+    @default_buildpack = nil
     def self.default_buildpack
-      [HATCHET_BUILDPACK_BASE, HATCHET_BUILDPACK_BRANCH.call].join("#")
+      @default_buildpack ||= [HATCHET_BUILDPACK_BASE.call, HATCHET_BUILDPACK_BRANCH.call].join("#")
     end
 
     def allow_failure?
@@ -157,32 +187,17 @@ module Hatchet
         command = command.to_s
       end
 
-      heroku_command = build_heroku_command(command, options)
-
       allow_run_multi! if @run_multi
 
-      output = ""
-
-      ShellThrottle.new(platform_api: @platform_api).call do |throttle|
-        output = `#{heroku_command}`
-        throw(:throttle) if output.match?(/reached the API rate limit/)
-      end
-
-      return output
-    end
-
-    private def build_heroku_command(command, options = {})
-      command = command.shellescape unless options.delete(:raw)
-
-      default_options = { "app" => name, "exit-code" => nil }
-      heroku_options = (default_options.merge(options.delete(:heroku) || {})).map do |k,v|
-        next if v == Hatchet::App::SkipDefaultOption # for forcefully removing e.g. --exit-code, a user can pass this
-        arg = "--#{k.to_s.shellescape}"
-        arg << "=#{v.to_s.shellescape}" unless v.nil? # nil means we include the option without an argument
-        arg
-      end.join(" ")
+      run_obj = Hatchet::HerokuRun.new(
+        command,
+        app: self,
+        retry_on_empty: options.fetch(:retry_on_empty, !ENV["HATCHET_DISABLE_EMPTY_RUN_RETRY"]),
+        heroku: options[:heroku],
+        raw: options[:raw]
+      ).call
 
-      "heroku run #{heroku_options} -- #{command}"
+      return run_obj.output
     end
 
     private def allow_run_multi!
@@ -191,7 +206,6 @@ module Hatchet
       @run_multi_is_setup ||= platform_api.formation.update(name, "web", {"size" => "Standard-1X"})
     end
 
-
     # Allows multiple commands to be run concurrently in the background.
     #
     # WARNING! Using the feature requres that the underlying app is not on the "free" Heroku
@@ -229,23 +243,15 @@ module Hatchet
       allow_run_multi!
 
       run_thread = Thread.new do
-        heroku_command = build_heroku_command(command, options)
-
-        out = nil
-        status = nil
-        ShellThrottle.new(platform_api: @platform_api).call do |throttle|
-          out = `#{heroku_command}`
-          throw(:throttle) if output.match?(/reached the API rate limit/)
-          status = $?
-        end
-
-        yield out, status
-
-        # if block.arity == 1
-        #   block.call(out)
-        # else
-        #   block.call(out, status)
-        # end
+        run_obj = Hatchet::HerokuRun.new(
+          command,
+          app: self,
+          retry_on_empty: options.fetch(:retry_on_empty, !ENV["HATCHET_DISABLE_EMPTY_RUN_RETRY"]),
+          heroku: options[:heroku],
+          raw: options[:raw]
+        ).call
+
+        yield run_obj.output, run_obj.status
       end
       run_thread.abort_on_exception = true
 
@@ -318,9 +324,25 @@ module Hatchet
       end
     end
 
-    def before_deploy(&block)
+    def before_deploy(behavior = :default, &block)
       raise "block required" unless block
-      @before_deploy = block
+
+      case behavior
+      when :default, :replace
+        if @before_deploy_array.any? && behavior == :default
+          STDERR.puts "Calling App#before_deploy multiple times will overwrite the contents. If you intended this: use `App#before_deploy(:replace)`"
+          STDERR.puts "In the future, calling this method with no arguements will default to `App#before_deploy(:append)` behavior.\n#{caller.join("\n")}"
+        end
+
+        @before_deploy_array.clear
+        @before_deploy_array << block
+      when :prepend
+        @before_deploy_array = [block] + @before_deploy_array
+      when :append
+        @before_deploy_array << block
+      else
+        raise "Unrecognized behavior: #{behavior.inspect}, valid inputs are :append, :prepend, and :replace"
+      end
 
       self
     end
@@ -346,14 +368,14 @@ module Hatchet
       @reaper.cycle if @app_is_setup
     end
 
-    def in_directory(directory = self.directory)
-      yield directory and return if @already_in_dir
+    def in_directory
+      yield and return if @already_in_dir
 
       Dir.mktmpdir do |tmpdir|
-        FileUtils.cp_r("#{directory}/.", "#{tmpdir}/.")
+        FileUtils.cp_r("#{original_source_code_directory}/.", "#{tmpdir}/.")
         Dir.chdir(tmpdir) do
           @already_in_dir = true
-          yield directory
+          yield
           @already_in_dir = false
         end
       end
@@ -366,8 +388,8 @@ module Hatchet
     def in_directory_fork(&block)
       Tempfile.create("stdout") do |tmp_file|
         pid = fork do
-          $stdout.reopen(tmp_file, "w")
-          $stderr.reopen(tmp_file, "w")
+          $stdout.reopen(tmp_file, "a")
+          $stderr.reopen(tmp_file, "a")
           $stdout.sync = true
           $stderr.sync = true
           in_directory do |dir|
@@ -378,7 +400,7 @@ module Hatchet
         Process.waitpid(pid)
 
         if $?.success?
-          puts File.read(tmp_file)
+          print File.read(tmp_file)
         else
           raise File.read(tmp_file)
         end
@@ -387,9 +409,11 @@ module Hatchet
 
     def deploy(&block)
       in_directory do
-        in_dir_setup!
-        self.push_with_retry!
-        block.call(self, api_rate_limit.call, output) if block_given?
+        annotate_failures do
+          in_dir_setup!
+          self.push_with_retry!
+          block.call(self, api_rate_limit.call, output) if block_given?
+        end
       end
     ensure
       self.teardown! if block_given?
@@ -423,21 +447,21 @@ module Hatchet
     end
 
     def api_key
-      @api_key ||= ENV['HEROKU_API_KEY'] || `heroku auth:token`.chomp
+      @api_key ||= ENV['HEROKU_API_KEY'] ||= `heroku auth:token 2> /dev/null`.chomp
     end
 
     def heroku
       raise "Not supported, use `platform_api` instead."
     end
 
-    def run_ci(timeout: 300, &block)
+    def run_ci(timeout: 900, &block)
       in_directory do
         max_retries_count.times.retry do
           result       = create_pipeline
           @pipeline_id = result["id"]
         end
 
-        # when the CI run finishes, the associated ephemeral app created for the test run internally gets removed almost immediately
+        # When the CI run finishes, the associated ephemeral app created for the test run internally gets removed almost immediately
         # the system then sees a pipeline with no apps, and deletes it, also almost immediately
         # that would, with bad timing, mean our test run info poll in wait! would 403, and/or the delete_pipeline at the end
         # that's why we create an app explictly (or maybe it already exists), and then associate it with with the pipeline
@@ -495,6 +519,9 @@ module Hatchet
 
     def delete_pipeline(pipeline_id)
       api_rate_limit.call.pipeline.delete(pipeline_id)
+    rescue Excon::Error::Forbidden
+      warn "Error deleting pipeline id: #{pipeline_id.inspect}, status: 403"
+      # Means the pipeline likely doesn't exist, not sure why though
     end
 
     def platform_api
@@ -534,14 +561,17 @@ module Hatchet
     end
 
     private def call_before_deploy
-      return unless @before_deploy
-      raise "before_deploy: #{@before_deploy.inspect} must respond to :call"  unless @before_deploy.respond_to?(:call)
-      raise "before_deploy: #{@before_deploy.inspect} must respond to :arity" unless @before_deploy.respond_to?(:arity)
+      return unless @before_deploy_array.any?
 
-      if @before_deploy.arity == 1
-        @before_deploy.call(self)
-      else
-        @before_deploy.call
+      @before_deploy_array.each do |block|
+        raise "before_deploy: #{block.inspect} must respond to :call"  unless block.respond_to?(:call)
+        raise "before_deploy: #{block.inspect} must respond to :arity" unless block.respond_to?(:arity)
+
+        if block.arity == 1
+          block.call(self)
+        else
+          block.call
+        end
       end
 
       commit! if needs_commit?