heroku_hatchet 5.0.2 → 7.1.1

data/bin/hatchet

@@ -46,7 +46,7 @@ class HatchetCLI < Thor
       Threaded.later do
         commit = lock_hash[directory]
         directory = File.expand_path(directory)
-        if Dir[directory].present?
+        if !Dir[directory]&.empty?
           puts "== pulling '#{git_repo}' into '#{directory}'\n"
           pull(directory, git_repo)
         else
@@ -67,6 +67,7 @@ class HatchetCLI < Thor
   desc "locks to specific git commits", "updates hatchet.lock"
   def lock
     lock_hash = {}
+    lockfile_hash = load_lockfile(create_if_does_not_exist: true)
     dirs.map do |directory, git_repo|
       Threaded.later do
         puts "== locking #{directory}"
@@ -75,8 +76,14 @@ class HatchetCLI < Thor
           clone(directory, git_repo, quiet: false)
         end
 
-        commit = commit_at_directory(directory)
-        lock_hash[directory] = commit
+        if lockfile_hash[directory] == "master"
+          lock_hash[directory] = "master"
+        elsif lockfile_hash[directory] == "main"
+          lock_hash[directory] = "main"
+        else
+          commit = commit_at_directory(directory)
+          lock_hash[directory] = commit
+        end
       end
     end.map(&:join)
 
@@ -111,10 +118,15 @@ class HatchetCLI < Thor
   end
 
   private
-  def load_lockfile
+  def load_lockfile(create_if_does_not_exist: false)
     return YAML.safe_load(File.read('hatchet.lock')).to_h
   rescue Errno::ENOENT
-    raise "No such file found `hatchet.lock` please run `$ bundle exec hatchet lock`"
+    if create_if_does_not_exist
+      FileUtils.touch('hatchet.lock')
+      {}
+    else
+      raise "No such file found `hatchet.lock` please run `$ bundle exec hatchet lock`"
+    end
   end
 
   def bad_repo?(url)
@@ -144,7 +156,7 @@ class HatchetCLI < Thor
   end
 
   def checkout_commit(directory, commit)
-    cmd("cd #{directory} && git reset --hard #{commit}")
+    cmd("cd #{directory} && git fetch origin #{commit} && git checkout #{commit} && git checkout - && git reset --hard #{commit}")
   end
 
   def commit_at_directory(directory)
@@ -152,7 +164,7 @@ class HatchetCLI < Thor
   end
 
   def pull(path, git_repo, commit: false)
-    cmd("cd #{path} && git pull --rebase #{git_repo} master --quiet")
+    cmd("cd #{path} && git pull --rebase #{git_repo} --quiet")
   end
 
   def clone(path, git_repo, quiet: true)

data/etc/ci_setup.rb

@@ -1,6 +1,14 @@
 #!/usr/bin/env ruby
-require 'bundler'
 require 'shellwords'
+
+STDOUT.sync = true
+
+def run_cmd(command)
+  puts "== Running: #{command}"
+  result = `#{command}`
+  raise "Command failed: #{command.inspect}\nResult: #{result}" unless $?.success?
+end
+
 puts "== Setting Up CI =="
 
 netrc_file = "#{ENV['HOME']}/.netrc"
@@ -14,18 +22,14 @@ machine api.heroku.com
   login #{ENV.fetch('HEROKU_API_USER')}
   password #{ENV.fetch('HEROKU_API_KEY')}
 EOF
-    `chmod 0600 "$HOME/.netrc"`
+    run_cmd 'chmod 0600 "$HOME/.netrc"'
   end
 end
 
-[
- "bundle exec hatchet ci:install_heroku",
- "bundle exec hatchet install",
- "git config --get user.email > /dev/null || git config --global user.email #{ENV.fetch('HEROKU_API_USER').shellescape}",
- "git config --get user.name > /dev/null || git config --global user.name 'BuildpackTester'",
-].each do |command|
-  puts "== Running: #{command}"
-  result = `#{command}`
-  raise "Command failed: #{command.inspect}\nResult: #{result}" unless $?.success?
-end
+run_cmd "bundle exec hatchet ci:install_heroku"
+run_cmd "bundle exec hatchet install"
+run_cmd "git config --get user.email > /dev/null || git config --global user.email #{ENV.fetch('HEROKU_API_USER').shellescape}"
+run_cmd "git config --get user.name > /dev/null || git config --global user.name 'BuildpackTester'"
+
 puts "== Done =="
+

data/etc/setup_heroku.sh

@@ -1,6 +1,4 @@
 #!/usr/bin/env bash
 
 set -euo pipefail
-sudo apt-get -qq update
-sudo apt-get install software-properties-common
 curl --fail --retry 3 --retry-delay 1 --connect-timeout 3 --max-time 30 https://cli-assets.heroku.com/install-ubuntu.sh | sh

data/hatchet.gemspec

@@ -18,18 +18,16 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "platform-api",  "~> 2"
+  gem.add_dependency "platform-api",  "~> 3"
   gem.add_dependency "rrrretry",      "~> 1"
   gem.add_dependency "excon",         "~> 0"
   gem.add_dependency "thor",          "~> 0"
-  gem.add_dependency "repl_runner",   "~> 0.0.3"
   gem.add_dependency "threaded",      "~> 0"
-  gem.add_dependency 'minitest-retry', '~> 0.1.9'
 
-  gem.add_development_dependency "minitest",       ">= 5.1"
+  gem.add_development_dependency "rspec"
   gem.add_development_dependency "rake",           ">= 10"
   gem.add_development_dependency "mocha",          ">= 1"
-  gem.add_development_dependency "parallel_tests", ">= 2"
+  gem.add_development_dependency "parallel_split_test"
   gem.add_development_dependency "travis",         ">= 1"
-  gem.add_development_dependency "m"
+  gem.add_development_dependency "rspec-retry"
 end

data/hatchet.json

@@ -1,6 +1,6 @@
 {
   "hatchet": {
-    "directory": "test/fixtures"
+    "directory": "repo_fixtures"
   },
   "rails3":  ["sharpstone/rails3_mri_193"],
   "rails2":  ["sharpstone/rails2blog"],
@@ -10,5 +10,9 @@
               "sharpstone/rails5_ruby_schema_format",
               "sharpstone/rails5_ci_fails_no_database"
  ],
- "lock": ["sharpstone/lock_fail"]
+ "lock": [
+   "sharpstone/lock_fail",
+   "sharpstone/lock_fail_master",
+   "sharpstone/lock_fail_main"
+ ]
 }

data/hatchet.lock

@@ -1,15 +1,19 @@
 ---
-- - test/fixtures/repos/bundler/no_lockfile
+- - repo_fixtures/repos/bundler/no_lockfile
   - 1947ce9a9c276d5df1c323b2ad78d1d85c7ab4c0
-- - test/fixtures/repos/ci/rails5_ci_fails_no_database
+- - repo_fixtures/repos/ci/rails5_ci_fails_no_database
   - 3044f05febdfbbe656f0f5113cf5968ca07e34fd
-- - test/fixtures/repos/ci/rails5_ruby_schema_format
-  - 3e63c3e13f435cf4ab11265e9abd161cc28cc552
-- - test/fixtures/repos/default/default_ruby
+- - repo_fixtures/repos/ci/rails5_ruby_schema_format
+  - d76be86c66ae8f45ec611fb2c4d8eb3adac0ad4b
+- - repo_fixtures/repos/default/default_ruby
   - 6e642963acec0ff64af51bd6fba8db3c4176ed6e
-- - test/fixtures/repos/lock/lock_fail
+- - repo_fixtures/repos/lock/lock_fail
   - da748a59340be8b950e7bbbfb32077eb67d70c3c
-- - test/fixtures/repos/rails2/rails2blog
+- - repo_fixtures/repos/lock/lock_fail_main
+  - main
+- - repo_fixtures/repos/lock/lock_fail_master
+  - master
+- - repo_fixtures/repos/rails2/rails2blog
   - b37357a498ae5e8429f5601c5ab9524021dc2aaa
-- - test/fixtures/repos/rails3/rails3_mri_193
+- - repo_fixtures/repos/rails3/rails3_mri_193
   - 88c5d0d067cfd11e4452633994a85b04627ae8c7

data/lib/hatchet.rb

@@ -1,4 +1,3 @@
-require 'active_support/core_ext/object/blank'
 require 'rrrretry'
 
 require 'json'
@@ -8,7 +7,7 @@ require 'stringio'
 require 'date'
 
 module Hatchet
-    APP_PREFIX = (ENV['HATCHET_APP_PREFIX'] || "hatchet-t-")
+  APP_PREFIX = (ENV['HATCHET_APP_PREFIX'] || "hatchet-t-")
 end
 
 require 'hatchet/version'

data/lib/hatchet/api_rate_limit.rb

@@ -1,14 +1,11 @@
-# Wraps platform-api and adds API rate limits
+# Legacy class
 #
-# Instead of:
-#
-#     platform_api.pipeline.create(name: @name)
-#
-# Use:
-#
-#     api_rate_limit = ApiRateLimit.new(platform_api)
-#     api_rate_limit.call.pipeline.create(name: @name)
+# Not needed since rate throttling went directly into the platform-api gem.
+# This class is effectively now a no-op
 #
+# It's being left in as it's interface was public and it's hard-ish to
+# deprecate/remove. Since it's so small there's not much value in removal
+# so it's probably fine to keep around for quite some time.
 class ApiRateLimit
   def initialize(platform_api)
     @platform_api = platform_api
@@ -16,24 +13,16 @@ class ApiRateLimit
     @called   = 0
   end
 
-
-  # Sleeps for progressively longer when api rate limit capacity
-  # is lower.
-  #
-  # Unfortunatley `@platform_api.rate_limit` is an extra API
-  # call, so by checking our limit, we also are using our limit 😬
-  # to partially mitigate this, only check capacity every 5
-  # api calls, or if the current capacity is under 1000
   def call
-    @called += 1
+    # @called += 1
 
-    if @called > 5 || @capacity < 1000
-      @called = 0
-      @capacity = @platform_api.rate_limit.info["remaining"]
-    end
+    # if @called > 5 || @capacity < 1000
+    #   @called = 0
+    #   @capacity = @platform_api.rate_limit.info["remaining"]
+    # end
 
-    sleep_time = (60/@capacity) if @capacity > 0.1 # no divide by zero
-    sleep(sleep_time || 60)
+    # sleep_time = (60/@capacity) if @capacity > 0.1 # no divide by zero
+    # sleep(sleep_time || 60)
 
     return @platform_api
   end

data/lib/hatchet/app.rb

@@ -9,14 +9,34 @@ module Hatchet
     HATCHET_BUILDPACK_BRANCH = -> { ENV['HATCHET_BUILDPACK_BRANCH'] || ENV['HEROKU_TEST_RUN_BRANCH'] || Hatchet.git_branch }
     BUILDPACK_URL = "https://github.com/heroku/heroku-buildpack-ruby.git"
 
-    attr_reader :name, :stack, :directory, :repo_name, :app_config, :buildpacks
-
-    class FailedDeploy < StandardError
-      def initialize(app, output)
-        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n" <<
-              " if this was expected add `allow_failure: true` to your deploy hash.\n" <<
-              "output:\n" <<
-              "#{output}"
+    attr_reader :name, :stack, :directory, :repo_name, :app_config, :buildpacks, :reaper, :max_retries_count
+
+    class FailedDeploy < StandardError; end
+
+    class FailedDeployError < FailedDeploy
+      attr_reader :output
+
+      def initialize(app, message, output: )
+        @output = output
+        msg = "Could not deploy '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
+        msg << "#{message}\n"
+        msg << "output:\n"
+        msg << "#{output}"
+        super(msg)
+      end
+    end
+
+    class FailedReleaseError < FailedDeploy
+      attr_reader :output
+
+      def initialize(app, message, output: )
+        @output = output
+        msg = "Could not release '#{app.name}' (#{app.repo_name}) using '#{app.class}' at path: '#{app.directory}'\n"
+        msg << "if this was expected add `allow_failure: true` to your deploy hash.\n"
+        msg << "#{message}\n"
+        msg << "output:\n"
+        msg << "#{output}"
         super(msg)
       end
     end
@@ -34,6 +54,8 @@ module Hatchet
                    buildpacks: nil,
                    buildpack_url: nil,
                    before_deploy: nil,
+                   run_multi: ENV["HATCHET_RUN_MULTI"],
+                   retries: RETRIES,
                    config: {}
                   )
       @repo_name     = repo_name
@@ -46,6 +68,13 @@ module Hatchet
       @buildpacks    = buildpack || buildpacks || buildpack_url || self.class.default_buildpack
       @buildpacks    = Array(@buildpacks)
       @buildpacks.map! {|b| b == :default ? self.class.default_buildpack : b}
+      @run_multi = run_multi
+      @max_retries_count = retries
+
+      if run_multi && !ENV["HATCHET_EXPENSIVE_MODE"]
+        raise "You're attempting to enable `run_multi: true` mode, but have not enabled `HATCHET_EXPENSIVE_MODE=1` env var to verify you understand the risks"
+      end
+      @run_multi_array = []
       @already_in_dir = nil
       @app_is_setup = nil
 
@@ -102,7 +131,7 @@ module Hatchet
     end
 
     def add_database(plan_name = 'heroku-postgresql:dev', match_val = "HEROKU_POSTGRESQL_[A-Z]+_URL")
-      Hatchet::RETRIES.times.retry do
+      max_retries_count.times.retry do
         # heroku.post_addon(name, plan_name)
         api_rate_limit.call.addon.create(name, plan: plan_name )
         _, value = get_config.detect {|k, v| k.match(/#{match_val}/) }
@@ -127,6 +156,22 @@ module Hatchet
       else
         command = command.to_s
       end
+
+      heroku_command = build_heroku_command(command, options)
+
+      allow_run_multi! if @run_multi
+
+      output = ""
+
+      ShellThrottle.new(platform_api: @platform_api).call do |throttle|
+        output = `#{heroku_command}`
+        throw(:throttle) if output.match?(/reached the API rate limit/)
+      end
+
+      return output
+    end
+
+    private def build_heroku_command(command, options = {})
       command = command.shellescape unless options.delete(:raw)
 
       default_options = { "app" => name, "exit-code" => nil }
@@ -136,16 +181,77 @@ module Hatchet
         arg << "=#{v.to_s.shellescape}" unless v.nil? # nil means we include the option without an argument
         arg
       end.join(" ")
-      heroku_command = "heroku run #{heroku_options} -- #{command}"
 
-      if block_given?
-        STDERR.puts "Using App#run with a block is deprecated, support for ReplRunner is being removed.\n#{caller}"
-        # When we deprecated this we can get rid of the "cmd_type" from the method signature
-        require 'repl_runner'
-        ReplRunner.new(cmd_type, heroku_command, options).run(&block)
-      else
-        `#{heroku_command}`
+      "heroku run #{heroku_options} -- #{command}"
+    end
+
+    private def allow_run_multi!
+      raise "Must explicitly enable the `run_multi: true` option. This requires scaling up a dyno and is not free, it may incur charges on your account" unless @run_multi
+
+      @run_multi_is_setup ||= platform_api.formation.update(name, "web", {"size" => "Standard-1X"})
+    end
+
+
+    # Allows multiple commands to be run concurrently in the background.
+    #
+    # WARNING! Using the feature requres that the underlying app is not on the "free" Heroku
+    # tier. This requires scaling up the dyno which is not free. If an app is
+    # scaled up and left in that state it can incur large costs.
+    #
+    # Enabling this feature should be done with extreme caution.
+    #
+    # Example:
+    #
+    #    Hatchet::Runner.new("default_ruby", run_multi: true)
+    #      app.run_multi("ls") { |out| expect(out).to include("Gemfile") }
+    #      app.run_multi("ruby -v") { |out| expect(out).to include("ruby") }
+    #    end
+    #
+    # This example will run `heroku run ls` as well as `ruby -v` at the same time in the background.
+    # The return result will be yielded to the block after they finish running.
+    #
+    # Order of execution is not guaranteed.
+    #
+    # If you need to assert a command was successful, you can yield a second status object like this:
+    #
+    #    Hatchet::Runner.new("default_ruby", run_multi: true)
+    #      app.run_multi("ls") do |out, status|
+    #        expect(status.success?).to be_truthy
+    #        expect(out).to include("Gemfile")
+    #      end
+    #      app.run_multi("ruby -v") do |out, status|
+    #        expect(status.success?).to be_truthy
+    #        expect(out).to include("ruby")
+    #      end
+    #    end
+    def run_multi(command, options = {}, &block)
+      raise "Block required" if block.nil?
+      allow_run_multi!
+
+      run_thread = Thread.new do
+        heroku_command = build_heroku_command(command, options)
+
+        out = nil
+        status = nil
+        ShellThrottle.new(platform_api: @platform_api).call do |throttle|
+          out = `#{heroku_command}`
+          throw(:throttle) if output.match?(/reached the API rate limit/)
+          status = $?
+        end
+
+        yield out, status
+
+        # if block.arity == 1
+        #   block.call(out)
+        # else
+        #   block.call(out, status)
+        # end
       end
+      run_thread.abort_on_exception = true
+
+      @run_multi_array << run_thread
+
+      true
     end
 
     # set debug: true when creating app if you don't want it to be
@@ -162,24 +268,26 @@ module Hatchet
     alias :no_debug? :not_debugging?
 
     def deployed?
-      # !heroku.get_ps(name).body.detect {|ps| ps["process"].include?("web") }.nil?
       api_rate_limit.call.formation.list(name).detect {|ps| ps["type"] == "web"}
     end
 
     def create_app
       3.times.retry do
         begin
-          # heroku.post_app({ name: name, stack: stack }.delete_if {|k,v| v.nil? })
           hash = { name: name, stack: stack }
           hash.delete_if { |k,v| v.nil? }
-          api_rate_limit.call.app.create(hash)
+          heroku_api_create_app(hash)
         rescue => e
-          @reaper.cycle
+          @reaper.cycle(app_exception_message: e.message)
           raise e
         end
       end
     end
 
+    private def heroku_api_create_app(hash)
+      api_rate_limit.call.app.create(hash)
+    end
+
     def update_stack(stack_name)
       @stack = stack_name
       api_rate_limit.call.app.update(name, build_stack: @stack)
@@ -210,7 +318,7 @@ module Hatchet
     end
 
     def commit!
-      local_cmd_exec!('git add .; git commit -m next')
+      local_cmd_exec!('git add .; git commit --allow-empty -m next')
     end
 
     def push_without_retry!
@@ -219,11 +327,15 @@ module Hatchet
 
     def teardown!
       return false unless @app_is_setup
-      if debugging?
-        puts "Debugging App:#{name}"
-        return false
+
+      if @run_multi_is_setup
+        @run_multi_array.map(&:join)
+        platform_api.formation.update(name, "web", {"size" => "free"})
       end
-      @reaper.cycle
+
+    ensure
+      @app_update_info = platform_api.app.update(name, { maintenance: true }) if @app_is_setup
+      @reaper.cycle if @app_is_setup
     end
 
     def in_directory(directory = self.directory)
@@ -265,10 +377,6 @@ module Hatchet
       end
     end
 
-    # creates a new app on heroku, "pushes" via anvil or git
-    # then yields to self so you can call self.run or
-    # self.deployed?
-    # Allow deploy failures on CI server by setting ENV['HATCHET_RETRIES']
     def deploy(&block)
       in_directory do
         self.setup!
@@ -276,16 +384,16 @@ module Hatchet
         block.call(self, api_rate_limit.call, output) if block_given?
       end
     ensure
-      self.teardown!
+      self.teardown! if block_given?
     end
 
     def push
-      max_retries = @allow_failure ? 1 : RETRIES
-      max_retries.times.retry do |attempt|
+      retry_count = allow_failure? ? 1 : max_retries_count
+      retry_count.times.retry do |attempt|
         begin
           @output = self.push_without_retry!
         rescue StandardError => error
-          puts retry_error_message(error, attempt, max_retries)
+          puts retry_error_message(error, attempt) unless retry_count == 1
           raise error
         end
       end
@@ -294,10 +402,10 @@ module Hatchet
     alias :push_with_retry  :push
     alias :push_with_retry! :push_with_retry
 
-    def retry_error_message(error, attempt, max_retries)
+    def retry_error_message(error, attempt)
       attempt += 1
-      return "" if attempt == max_retries
-      msg = "\nRetrying failed Attempt ##{attempt}/#{max_retries} to push for '#{name}' due to error: \n"<<
+      return "" if attempt == max_retries_count
+      msg = "\nRetrying failed Attempt ##{attempt}/#{max_retries_count} to push for '#{name}' due to error: \n"<<
             "#{error.class} #{error.message}\n  #{error.backtrace.join("\n  ")}"
       return msg
     end
@@ -316,7 +424,7 @@ module Hatchet
 
     def run_ci(timeout: 300, &block)
       in_directory do
-        Hatchet::RETRIES.times.retry do
+        max_retries_count.times.retry do
           result       = create_pipeline
           @pipeline_id = result["id"]
         end
@@ -327,7 +435,7 @@ module Hatchet
         # that's why we create an app explictly (or maybe it already exists), and then associate it with with the pipeline
         # the app will be auto cleaned up later
         self.setup!
-        Hatchet::RETRIES.times.retry do
+        max_retries_count.times.retry do
           couple_pipeline(@name, @pipeline_id)
         end
 
@@ -340,12 +448,13 @@ module Hatchet
           api_rate_limit: api_rate_limit
         )
 
-        Hatchet::RETRIES.times.retry do
+        max_retries_count.times.retry do
           test_run.create_test_run
         end
         test_run.wait!(&block)
       end
     ensure
+      teardown! if block_given?
       delete_pipeline(@pipeline_id) if @pipeline_id
       @pipeline_id = nil
     end
@@ -381,7 +490,6 @@ module Hatchet
     end
 
     def platform_api
-      puts "Deprecated: use `api_rate_limit.call` instead of platform_api"
       api_rate_limit
       return @platform_api
     end
@@ -433,3 +541,4 @@ module Hatchet
   end
 end
 
+require_relative 'shell_throttle.rb'