mrsk 0.11.0 → 0.13.2

data/lib/mrsk/cli/healthcheck.rb

@@ -1,7 +1,4 @@
 class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
-
-  class HealthcheckError < StandardError; end
-
   default_command :perform
 
   desc "perform", "Health check current app version"
@@ -9,38 +6,11 @@ class Mrsk::Cli::Healthcheck < Mrsk::Cli::Base
     on(MRSK.primary_host) do
       begin
         execute *MRSK.healthcheck.run
-
-        target = "Health check against #{MRSK.config.healthcheck["path"]}"
-        attempt = 1
-        max_attempts = MRSK.config.healthcheck["max_attempts"]
-
-        begin
-          status = capture_with_info(*MRSK.healthcheck.curl)
-
-          if status == "200"
-            info "#{target} succeeded with 200 OK!"
-          else
-            raise HealthcheckError, "#{target} failed with status #{status}"
-          end
-        rescue SSHKit::Command::Failed
-          if attempt <= max_attempts
-            info "#{target} failed to respond, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-            sleep attempt
-            attempt += 1
-
-            retry
-          else
-            raise
-          end
-        end
-      rescue SSHKit::Command::Failed, HealthcheckError => e
+        Mrsk::Utils::HealthcheckPoller.wait_for_healthy { capture_with_info(*MRSK.healthcheck.status) }
+      rescue Mrsk::Utils::HealthcheckPoller::HealthcheckError => e
         error capture_with_info(*MRSK.healthcheck.logs)
-
-        if e.message =~ /curl/
-          raise SSHKit::Command::Failed, "#{target} failed to return 200 OK!"
-        else
-          raise
-        end
+        error capture_with_pretty_json(*MRSK.healthcheck.container_health_log)
+        raise
       ensure
         execute *MRSK.healthcheck.stop, raise_on_non_zero_exit: false
         execute *MRSK.healthcheck.remove, raise_on_non_zero_exit: false

data/lib/mrsk/cli/lock.rb

@@ -2,16 +2,16 @@ class Mrsk::Cli::Lock < Mrsk::Cli::Base
   desc "status", "Report lock status"
   def status
     handle_missing_lock do
-      on(MRSK.primary_host) { puts capture_with_info(*MRSK.lock.status) }
+      on(MRSK.primary_host) { puts capture_with_debug(*MRSK.lock.status) }
     end
   end
 
   desc "acquire", "Acquire the deploy lock"
-  option :message, aliases: "-m", type: :string, desc: "A lock mesasge", required: true
+  option :message, aliases: "-m", type: :string, desc: "A lock message", required: true
   def acquire
     message = options[:message]
-    handle_missing_lock do
-      on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) }
+    raise_if_locked do
+      on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version), verbosity: :debug }
       say "Acquired the deploy lock"
     end
   end
@@ -19,7 +19,7 @@ class Mrsk::Cli::Lock < Mrsk::Cli::Base
   desc "release", "Release the deploy lock"
   def release
     handle_missing_lock do
-      on(MRSK.primary_host) { execute *MRSK.lock.release }
+      on(MRSK.primary_host) { execute *MRSK.lock.release, verbosity: :debug }
       say "Released the deploy lock"
     end
   end

data/lib/mrsk/cli/main.rb

@@ -1,8 +1,8 @@
 class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "setup", "Setup all accessories and deploy app to servers"
   def setup
-    with_lock do
-      print_runtime do
+    print_runtime do
+      with_lock do
         invoke "mrsk:cli:server:bootstrap"
         invoke "mrsk:cli:accessory:boot", [ "all" ]
         deploy
@@ -13,12 +13,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "deploy", "Deploy app to servers"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def deploy
-    with_lock do
-      invoke_options = deploy_options
-
-      runtime = print_runtime do
-        say "Ensure curl and Docker are installed...", :magenta
-        invoke "mrsk:cli:server:bootstrap", [], invoke_options
+    runtime = print_runtime do
+      with_lock do
+        invoke_options = deploy_options
 
         say "Log into image registry...", :magenta
         invoke "mrsk:cli:registry:login", [], invoke_options
@@ -31,29 +28,34 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
           invoke "mrsk:cli:build:deliver", [], invoke_options
         end
 
+        run_hook "pre-deploy"
+
         say "Ensure Traefik is running...", :magenta
         invoke "mrsk:cli:traefik:boot", [], invoke_options
 
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
 
         say "Prune old containers and images...", :magenta
         invoke "mrsk:cli:prune:all", [], invoke_options
       end
-
-      audit_broadcast "Deployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
     end
+
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
-    with_lock do
-      invoke_options = deploy_options
+    runtime = print_runtime do
+      with_lock do
+        invoke_options = deploy_options
 
-      runtime = print_runtime do
         if options[:skip_push]
           say "Pull app image...", :magenta
           invoke "mrsk:cli:build:pull", [], invoke_options
@@ -62,49 +64,43 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
           invoke "mrsk:cli:build:deliver", [], invoke_options
         end
 
+        run_hook "pre-deploy"
+
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
         invoke "mrsk:cli:app:boot", [], invoke_options
       end
-
-      audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
     end
+
+    run_hook "post-deploy", runtime: runtime.round
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
-    with_lock do
-      MRSK.config.version = version
+    rolled_back = false
+    runtime = print_runtime do
+      with_lock do
+        invoke_options = deploy_options
 
-      if container_available?(version)
-        say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
-
-        cli = self
+        MRSK.config.version = version
         old_version = nil
 
-        on(MRSK.hosts) do |host|
-          roles = MRSK.roles_on(host)
-
-          roles.each do |role|
-            app = MRSK.app(role: role)
-            old_version = capture_with_info(*app.current_running_version).strip.presence
+        if container_available?(version)
+          run_hook "pre-deploy"
 
-            execute *app.start
-
-            if old_version
-              sleep MRSK.config.readiness_delay
-
-              execute *app.stop(version: old_version), raise_on_non_zero_exit: false
-            end
-          end
+          invoke "mrsk:cli:app:boot", [], invoke_options.merge(version: version)
+          rolled_back = true
+        else
+          say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
         end
-
-        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
-      else
-        say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
       end
     end
+
+    run_hook "post-deploy", runtime: runtime.round if rolled_back
   end
 
   desc "details", "Show details about all containers"
@@ -146,6 +142,14 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       puts "Created .env file"
     end
 
+    unless (hooks_dir = Pathname.new(File.expand_path(".mrsk/hooks"))).exist?
+      hooks_dir.mkpath
+      Pathname.new(File.expand_path("templates/sample_hooks", __dir__)).each_child do |sample_hook|
+        FileUtils.cp sample_hook, hooks_dir, preserve: true
+      end
+      puts "Created sample hooks in .mrsk/hooks"
+    end
+
     if options[:bundle]
       if (binstub = Pathname.new(File.expand_path("bin/mrsk"))).exist?
         puts "Binstub already exists in bin/mrsk (remove first to create a new one)"
@@ -203,6 +207,9 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "healthcheck", "Healthcheck application"
   subcommand "healthcheck", Mrsk::Cli::Healthcheck
 
+  desc "lock", "Manage the deploy lock"
+  subcommand "lock", Mrsk::Cli::Lock
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Mrsk::Cli::Prune
 
@@ -215,26 +222,28 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "traefik", "Manage Traefik load balancer"
   subcommand "traefik", Mrsk::Cli::Traefik
 
-  desc "lock", "Manage the deploy lock"
-  subcommand "lock", Mrsk::Cli::Lock
-
   private
-    def container_available?(version, host: MRSK.primary_host)
-      available = nil
-
-      on(host) do
-        first_role = MRSK.roles_on(host).first
-        available = capture_with_info(*MRSK.app(role: first_role).container_id_for_version(version)).present?
+    def container_available?(version)
+      begin
+        on(MRSK.hosts) do
+          MRSK.roles_on(host).each do |role|
+            container_id = capture_with_info(*MRSK.app(role: role).container_id_for_version(version))
+            raise "Container not found" unless container_id.present?
+          end
+        end
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /Container not found/
+          say "Error looking for container version #{version}: #{e.message}"
+          return false
+        else
+          raise
+        end
       end
 
-      available
+      true
     end
 
     def deploy_options
       { "version" => MRSK.config.version }.merge(options.without("skip_push"))
     end
-
-    def service_version(version = MRSK.config.abbreviated_version)
-      [ MRSK.config.service, version ].compact.join("@")
-    end
 end

data/lib/mrsk/cli/prune.rb

@@ -7,17 +7,18 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
     end
   end
 
-  desc "images", "Prune unused images older than 7 days"
+  desc "images", "Prune dangling images"
   def images
     with_lock do
       on(MRSK.hosts) do
         execute *MRSK.auditor.record("Pruned images"), verbosity: :debug
-        execute *MRSK.prune.images
+        execute *MRSK.prune.dangling_images
+        execute *MRSK.prune.tagged_images
       end
     end
   end
 
-  desc "containers", "Prune stopped containers older than 3 days"
+  desc "containers", "Prune all stopped containers, except the last 5"
   def containers
     with_lock do
       on(MRSK.hosts) do

data/lib/mrsk/cli/server.rb

@@ -1,17 +1,21 @@
 class Mrsk::Cli::Server < Mrsk::Cli::Base
-  desc "bootstrap", "Ensure curl and Docker are installed on servers"
+  desc "bootstrap", "Set up Docker to run MRSK apps"
   def bootstrap
-    with_lock do
-      on(MRSK.hosts + MRSK.accessory_hosts) do
-        dependencies_to_install = Array.new.tap do |dependencies|
-          dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
-          dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
-        end
+    missing = []
 
-        if dependencies_to_install.any?
-          execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
+    on(MRSK.hosts | MRSK.accessory_hosts) do |host|
+      unless execute(*MRSK.docker.installed?, raise_on_non_zero_exit: false)
+        if execute(*MRSK.docker.superuser?, raise_on_non_zero_exit: false)
+          info "Missing Docker on #{host}. Installing…"
+          execute *MRSK.docker.install
+        else
+          missing << host
         end
       end
     end
+
+    if missing.any?
+      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
+    end
   end
 end

data/lib/mrsk/cli/templates/deploy.yml

@@ -25,10 +25,6 @@ registry:
 #   secret:
 #     - RAILS_MASTER_KEY
 
-# Call a broadcast command on deploys.
-# audit_broadcast_cmd:
-#   bin/broadcast_to_bc
-
 # Use a different ssh user than root
 # ssh:
 #   user: app

data/lib/mrsk/cli/templates/sample_hooks/post-deploy.sample

@@ -0,0 +1,14 @@
+#!/bin/sh
+
+# A sample post-deploy hook
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_HOSTS
+# MRSK_ROLE (if set)
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+echo "$MRSK_PERFORMER deployed $MRSK_VERSION to $MRSK_DESTINATION in $MRSK_RUNTIME seconds"

data/lib/mrsk/cli/templates/sample_hooks/pre-build.sample

@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# A sample pre-build hook
+#
+# Checks:
+# 1. We have a clean checkout
+# 2. A remote is configured
+# 3. The branch has been pushed to the remote
+# 4. The version we are deploying matches the remote
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_HOSTS
+# MRSK_ROLE (if set)
+# MRSK_DESTINATION (if set)
+
+if [ -n "$(git status --porcelain)" ]; then
+  echo "Git checkout is not clean, aborting..." >&2
+  git status --porcelain >&2
+  exit 1
+fi
+
+first_remote=$(git remote)
+
+if [ -z "$first_remote" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+current_branch=$(git branch --show-current)
+
+if [ -z "$current_branch" ]; then
+  echo "No git remote set, aborting..." >&2
+  exit 1
+fi
+
+remote_head=$(git ls-remote $first_remote --tags $current_branch | cut -f1)
+
+if [ -z "$remote_head" ]; then
+  echo "Branch not pushed to remote, aborting..." >&2
+  exit 1
+fi
+
+if [ "$MRSK_VERSION" != "$remote_head" ]; then
+  echo "Version ($MRSK_VERSION) does not match remote HEAD ($remote_head), aborting..." >&2
+  exit 1
+fi
+
+exit 0

data/lib/mrsk/cli/templates/sample_hooks/pre-connect.sample

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+# A sample pre-connect check
+#
+# Warms DNS before connecting to hosts in parallel
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_HOSTS
+# MRSK_ROLE (if set)
+# MRSK_DESTINATION (if set)
+# MRSK_RUNTIME
+
+hosts = ENV["MRSK_HOSTS"].split(",")
+results = nil
+max = 3
+
+elapsed = Benchmark.realtime do
+  results = hosts.map do |host|
+    Thread.new do
+      tries = 1
+
+      begin
+        Socket.getaddrinfo(host, 0, Socket::AF_UNSPEC, Socket::SOCK_STREAM, nil, Socket::AI_CANONNAME)
+      rescue SocketError
+        if tries < max
+          puts "Retrying DNS warmup: #{host}"
+          tries += 1
+          sleep rand
+          retry
+        else
+          puts "DNS warmup failed: #{host}"
+          host
+        end
+      end
+
+      tries
+    end
+  end.map(&:value)
+end
+
+retries = results.sum - hosts.size
+nopes = results.count { |r| r == max }
+
+puts "Prewarmed %d DNS lookups in %.2f sec: %d retries, %d failures" % [ hosts.size, elapsed, retries, nopes ]

data/lib/mrsk/cli/templates/sample_hooks/pre-deploy.sample

@@ -0,0 +1,82 @@
+#!/bin/sh
+
+# A sample pre-deploy hook
+#
+# Checks the Github status of the build, waiting for a pending build to complete for up to 720 seconds.
+#
+# Fails unless the combined status is "success"
+#
+# These environment variables are available:
+# MRSK_RECORDED_AT
+# MRSK_PERFORMER
+# MRSK_VERSION
+# MRSK_HOSTS
+# MRSK_COMMAND
+# MRSK_SUBCOMMAND
+# MRSK_ROLE (if set)
+# MRSK_DESTINATION (if set)
+
+#!/usr/bin/env ruby
+
+# Only check the build status for production deployments
+if ENV["MRSK_COMMAND"] == "rollback" || ENV["MRSK_DESTINATION"] != "production"
+  exit 0
+end
+
+require "bundler/inline"
+
+# true = install gems so this is fast on repeat invocations
+gemfile(true, quiet: true) do
+  source "https://rubygems.org"
+
+  gem "octokit"
+  gem "faraday-retry"
+end
+
+MAX_ATTEMPTS = 72
+ATTEMPTS_GAP = 10
+
+def exit_with_error(message)
+  $stderr.puts message
+  exit 1
+end
+
+def first_status_url(combined_status, state)
+  first_status = combined_status[:statuses].find { |status| status[:state] == state }
+  first_status && first_status[:target_url]
+end
+
+remote_url = `git config --get remote.origin.url`.strip.delete_prefix("https://github.com/")
+git_sha = `git rev-parse HEAD`.strip
+
+repository = Octokit::Repository.from_url(remote_url)
+github_client = Octokit::Client.new(access_token: ENV["GITHUB_TOKEN"])
+attempts = 0
+
+begin
+  loop do
+    combined_status = github_client.combined_status(remote_url, git_sha)
+    state = combined_status[:state]
+    first_status_url = first_status_url(combined_status, state)
+
+    case state
+    when "success"
+      puts "Build passed, see #{first_status_url}"
+      exit 0
+    when "failure"
+      exit_with_error "Build failed, see #{first_status_url}"
+    when "pending"
+      attempts += 1
+    end
+
+    puts "Waiting #{ATTEMPTS_GAP} more seconds for build to complete#{", see #{first_status_url}" if first_status_url}..."
+
+    if attempts == MAX_ATTEMPTS
+      exit_with_error "Build status is still pending, gave up after #{MAX_ATTEMPTS * ATTEMPTS_GAP} seconds"
+    end
+
+    sleep(ATTEMPTS_GAP)
+  end
+rescue Octokit::NotFound
+  exit_with_error "Build status could not be found"
+end

data/lib/mrsk/cli/traefik.rb

@@ -94,7 +94,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
     end
   end
 
-  desc "remove_container", "Remove Traefik image from servers", hide: true
+  desc "remove_image", "Remove Traefik image from servers", hide: true
   def remove_image
     with_lock do
       on(MRSK.traefik_hosts) do

data/lib/mrsk/cli.rb

@@ -1,5 +1,6 @@
 module Mrsk::Cli
   class LockError < StandardError; end
+  class HookError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

data/lib/mrsk/commander.rb

@@ -2,11 +2,12 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Mrsk::Commander
-  attr_accessor :verbosity, :lock_count
+  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
 
   def initialize
     self.verbosity = :info
-    self.lock_count = 0
+    self.holding_lock = false
+    self.hold_lock_on_error = false
   end
 
   def config
@@ -35,7 +36,7 @@ class Mrsk::Commander
   end
 
   def primary_host
-    specific_hosts&.first || config.primary_web_host
+    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
   end
 
   def roles
@@ -50,6 +51,14 @@ class Mrsk::Commander
     end
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end
@@ -75,18 +84,30 @@ class Mrsk::Commander
     Mrsk::Commands::Accessory.new(config, name: name)
   end
 
-  def auditor(role: nil)
-    Mrsk::Commands::Auditor.new(config, role: role)
+  def auditor(**details)
+    Mrsk::Commands::Auditor.new(config, **details)
   end
 
   def builder
     @builder ||= Mrsk::Commands::Builder.new(config)
   end
 
+  def docker
+    @docker ||= Mrsk::Commands::Docker.new(config)
+  end
+
   def healthcheck
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
 
+  def hook
+    @hook ||= Mrsk::Commands::Hook.new(config)
+  end
+
+  def lock
+    @lock ||= Mrsk::Commands::Lock.new(config)
+  end
+
   def prune
     @prune ||= Mrsk::Commands::Prune.new(config)
   end
@@ -99,10 +120,6 @@ class Mrsk::Commander
     @traefik ||= Mrsk::Commands::Traefik.new(config)
   end
 
-  def lock
-    @lock ||= Mrsk::Commands::Lock.new(config)
-  end
-
   def with_verbosity(level)
     old_level = self.verbosity
 
@@ -115,6 +132,14 @@ class Mrsk::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def holding_lock?
+    self.holding_lock
+  end
+
+  def hold_lock_on_error?
+    self.hold_lock_on_error
+  end
+
   private
     # Lazy setup of SSHKit
     def configure_sshkit_with(config)