mrsk 0.10.1 → 0.12.0

data/lib/mrsk/cli/main.rb

@@ -17,9 +17,6 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
       invoke_options = deploy_options
 
       runtime = print_runtime do
-        say "Ensure curl and Docker are installed...", :magenta
-        invoke "mrsk:cli:server:bootstrap", [], invoke_options
-
         say "Log into image registry...", :magenta
         invoke "mrsk:cli:registry:login", [], invoke_options
 
@@ -37,7 +34,12 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        invoke "mrsk:cli:app:boot", [], invoke_options
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
+        hold_lock_on_error do
+          invoke "mrsk:cli:app:boot", [], invoke_options
+        end
 
         say "Prune old containers and images...", :magenta
         invoke "mrsk:cli:prune:all", [], invoke_options
@@ -65,7 +67,12 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
         say "Ensure app can pass healthcheck...", :magenta
         invoke "mrsk:cli:healthcheck:perform", [], invoke_options
 
-        invoke "mrsk:cli:app:boot", [], invoke_options
+        say "Detect stale containers...", :magenta
+        invoke "mrsk:cli:app:stale_containers", [], invoke_options
+
+        hold_lock_on_error do
+          invoke "mrsk:cli:app:boot", [], invoke_options
+        end
       end
 
       audit_broadcast "Redeployed #{service_version} in #{runtime.round} seconds" unless options[:skip_broadcast]
@@ -75,29 +82,41 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "rollback [VERSION]", "Rollback app to VERSION"
   def rollback(version)
     with_lock do
-      MRSK.config.version = version
-
-      if container_name_available?(MRSK.config.service_with_version)
-        say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
+      invoke_options = deploy_options
 
-        cli = self
+      hold_lock_on_error do
+        MRSK.config.version = version
         old_version = nil
 
-        on(MRSK.hosts) do |host|
-          old_version = capture_with_info(*MRSK.app.current_running_version).strip.presence
+        if container_available?(version)
+          say "Start version #{version}, then wait #{MRSK.config.readiness_delay}s for app to boot before stopping the old version...", :magenta
+
+          on(MRSK.hosts) do
+            execute *MRSK.auditor.record("Tagging #{MRSK.config.absolute_image} as the latest image"), verbosity: :debug
+            execute *MRSK.app.tag_current_as_latest
+          end
+
+          on(MRSK.hosts) do |host|
+            roles = MRSK.roles_on(host)
+
+            roles.each do |role|
+              app = MRSK.app(role: role)
+              old_version = capture_with_info(*app.current_running_version).strip.presence
 
-          execute *MRSK.app.start
+              execute *app.start
 
-          if old_version
-            sleep MRSK.config.readiness_delay
+              if old_version
+                sleep MRSK.config.readiness_delay
 
-            execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false
+                execute *app.stop(version: old_version), raise_on_non_zero_exit: false
+              end
+            end
           end
-        end
 
-        audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
-      else
-        say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+          audit_broadcast "Rolled back #{service_version(Mrsk::Utils.abbreviate_version(old_version))} to #{service_version}" unless options[:skip_broadcast]
+        else
+          say "The app version '#{version}' is not available as a container (use 'mrsk app containers' for available versions)", :red
+        end
       end
     end
   end
@@ -119,7 +138,7 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   desc "config", "Show combined config (including secrets!)"
   def config
     run_locally do
-      puts MRSK.config.to_h.to_yaml
+      puts Mrsk::Utils.redacted(MRSK.config.to_h).to_yaml
     end
   end
 
@@ -214,10 +233,24 @@ class Mrsk::Cli::Main < Mrsk::Cli::Base
   subcommand "lock", Mrsk::Cli::Lock
 
   private
-    def container_name_available?(container_name, host: MRSK.primary_host)
-      container_names = nil
-      on(host) { container_names = capture_with_info(*MRSK.app.list_container_names).split("\n") }
-      Array(container_names).include?(container_name)
+    def container_available?(version)
+      begin
+        on(MRSK.hosts) do
+          MRSK.roles_on(host).each do |role|
+            container_id = capture_with_info(*MRSK.app(role: role).container_id_for_version(version))
+            raise "Container not found" unless container_id.present?
+          end
+        end
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /Container not found/
+          say "Error looking for container version #{version}: #{e.message}"
+          return false
+        else
+          raise
+        end
+      end
+
+      true
     end
 
     def deploy_options

data/lib/mrsk/cli/prune.rb

@@ -7,7 +7,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
     end
   end
 
-  desc "images", "Prune unused images older than 7 days"
+  desc "images", "Prune dangling images"
   def images
     with_lock do
       on(MRSK.hosts) do
@@ -17,7 +17,7 @@ class Mrsk::Cli::Prune < Mrsk::Cli::Base
     end
   end
 
-  desc "containers", "Prune stopped containers older than 3 days"
+  desc "containers", "Prune all stopped containers, except the last 5"
   def containers
     with_lock do
       on(MRSK.hosts) do

data/lib/mrsk/cli/server.rb

@@ -1,17 +1,21 @@
 class Mrsk::Cli::Server < Mrsk::Cli::Base
-  desc "bootstrap", "Ensure curl and Docker are installed on servers"
+  desc "bootstrap", "Set up Docker to run MRSK apps"
   def bootstrap
-    with_lock do
-      on(MRSK.hosts + MRSK.accessory_hosts) do
-        dependencies_to_install = Array.new.tap do |dependencies|
-          dependencies << "curl" unless execute "which curl", raise_on_non_zero_exit: false
-          dependencies << "docker.io" unless execute "which docker", raise_on_non_zero_exit: false
-        end
+    missing = []
 
-        if dependencies_to_install.any?
-          execute "apt-get update -y && apt-get install #{dependencies_to_install.join(" ")} -y"
+    on(MRSK.hosts | MRSK.accessory_hosts) do |host|
+      unless execute(*MRSK.docker.installed?, raise_on_non_zero_exit: false)
+        if execute(*MRSK.docker.superuser?, raise_on_non_zero_exit: false)
+          info "Missing Docker on #{host}. Installing…"
+          execute *MRSK.docker.install
+        else
+          missing << host
         end
       end
     end
+
+    if missing.any?
+      raise "Docker is not installed on #{missing.join(", ")} and can't be automatically installed without having root access and the `curl` command available. Install Docker manually: https://docs.docker.com/engine/install/"
+    end
   end
 end

data/lib/mrsk/cli/traefik.rb

@@ -2,7 +2,10 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
   desc "boot", "Boot Traefik on servers"
   def boot
     with_lock do
-      on(MRSK.traefik_hosts) { execute *MRSK.traefik.run, raise_on_non_zero_exit: false }
+      on(MRSK.traefik_hosts) do
+        execute *MRSK.registry.login
+        execute *MRSK.traefik.run, raise_on_non_zero_exit: false
+      end
     end
   end
 
@@ -91,7 +94,7 @@ class Mrsk::Cli::Traefik < Mrsk::Cli::Base
     end
   end
 
-  desc "remove_container", "Remove Traefik image from servers", hide: true
+  desc "remove_image", "Remove Traefik image from servers", hide: true
   def remove_image
     with_lock do
       on(MRSK.traefik_hosts) do

data/lib/mrsk/cli.rb

@@ -1,4 +1,5 @@
 module Mrsk::Cli
+  class LockError < StandardError; end
 end
 
 # SSHKit uses instance eval, so we need a global const for ergonomics

data/lib/mrsk/commander.rb

@@ -2,11 +2,12 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Mrsk::Commander
-  attr_accessor :verbosity, :lock_count
+  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
 
   def initialize
     self.verbosity = :info
-    self.lock_count = 0
+    self.holding_lock = false
+    self.hold_lock_on_error = false
   end
 
   def config
@@ -35,7 +36,7 @@ class Mrsk::Commander
   end
 
   def primary_host
-    specific_hosts&.first || config.primary_web_host
+    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
   end
 
   def roles
@@ -50,6 +51,14 @@ class Mrsk::Commander
     end
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end
@@ -83,6 +92,10 @@ class Mrsk::Commander
     @builder ||= Mrsk::Commands::Builder.new(config)
   end
 
+  def docker
+    @docker ||= Mrsk::Commands::Docker.new(config)
+  end
+
   def healthcheck
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
@@ -115,6 +128,14 @@ class Mrsk::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def holding_lock?
+    self.holding_lock
+  end
+
+  def hold_lock_on_error?
+    self.hold_lock_on_error
+  end
+
   private
     # Lazy setup of SSHKit
     def configure_sshkit_with(config)

data/lib/mrsk/commands/app.rb

@@ -15,6 +15,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "--name", container_name,
       "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
       *role.env_args,
+      *role.health_check_args,
       *config.logging_args,
       *config.volume_args,
       *role.label_args,
@@ -27,9 +28,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :start, container_name
   end
 
+  def status(version:)
+    pipe container_id_for_version(version), xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
+  end
+
   def stop(version: nil)
     pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
   end
 
@@ -40,7 +45,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
-      current_container_id,
+      current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
@@ -48,7 +53,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh \
       pipe(
-        current_container_id,
+        current_running_container_id,
         "xargs docker logs --timestamps --tail 10 --follow 2>&1",
         (%(grep "#{grep}") if grep)
       ),
@@ -82,8 +87,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
 
-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
   end
 
   def container_id_for_version(version)
@@ -91,11 +96,14 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
     pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      %(cut -c 2-)
   end
 
   def list_containers
@@ -128,20 +136,25 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
+  def tag_current_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+
 
   private
     def container_name(version = nil)
       [ config.service, role, config.destination, version || config.version ].compact.join("-")
     end
 
-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
     end
 
-    def filters
+    def filters(status: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
         filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
       end
     end
 end

data/lib/mrsk/commands/base.rb

@@ -1,6 +1,8 @@
 module Mrsk::Commands
   class Base
-    delegate :redact, :argumentize, to: Mrsk::Utils
+    delegate :sensitive, :argumentize, to: Mrsk::Utils
+
+    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
 
     attr_accessor :config
 

data/lib/mrsk/commands/builder/base.rb

@@ -1,4 +1,7 @@
+
 class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
+  class BuilderError < StandardError; end
+
   delegate :argumentize, to: Mrsk::Utils
 
   def clean
@@ -7,7 +10,6 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
 
   def pull
     docker :pull, config.absolute_image
-    docker :pull, config.latest_image
   end
 
   def build_options
@@ -18,6 +20,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
     context
   end
 
+
   private
     def build_tags
       [ "-t", config.absolute_image, "-t", config.latest_image ]
@@ -28,7 +31,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
     end
 
     def build_args
-      argumentize "--build-arg", args, redacted: true
+      argumentize "--build-arg", args, sensitive: true
     end
 
     def build_secrets
@@ -36,7 +39,11 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
     end
 
     def build_dockerfile
-      argumentize "--file", dockerfile
+      if Pathname.new(File.expand_path(dockerfile)).exist?
+        argumentize "--file", dockerfile
+      else
+        raise BuilderError, "Missing #{dockerfile}"
+      end
     end
 
     def args

data/lib/mrsk/commands/builder.rb

@@ -2,7 +2,7 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
   delegate :create, :remove, :push, :clean, :pull, :info, to: :target
 
   def name
-    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore
+    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore.inquiry
   end
 
   def target
@@ -33,4 +33,24 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
   def multiarch_remote
     @multiarch_remote ||= Mrsk::Commands::Builder::Multiarch::Remote.new(config)
   end
+
+
+  def ensure_local_dependencies_installed
+    if name.native?
+      ensure_local_docker_installed
+    else
+      combine \
+        ensure_local_docker_installed,
+        ensure_local_buildx_installed
+    end
+  end
+
+  private
+    def ensure_local_docker_installed
+      docker "--version"
+    end
+
+    def ensure_local_buildx_installed
+      docker :buildx, "version"
+    end
 end

data/lib/mrsk/commands/docker.rb

@@ -0,0 +1,21 @@
+class Mrsk::Commands::Docker < Mrsk::Commands::Base
+  # Install Docker using the https://github.com/docker/docker-install convenience script.
+  def install
+    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+  end
+
+  # Checks the Docker client version. Fails if Docker is not installed.
+  def installed?
+    docker "-v"
+  end
+
+  # Checks the Docker server version. Fails if Docker is not running.
+  def running?
+    docker :version
+  end
+
+  # Do we have superuser access to install Docker and start system services?
+  def superuser?
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+  end
+end

data/lib/mrsk/commands/healthcheck.rb

@@ -11,14 +11,15 @@ class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
       "--label", "service=#{container_name}",
       "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
       *web.env_args,
+      *web.health_check_args,
       *config.volume_args,
       *web.option_args,
       config.absolute_image,
       web.cmd
   end
 
-  def curl
-    [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", health_url ]
+  def status
+    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
   end
 
   def logs

data/lib/mrsk/commands/prune.rb

@@ -2,11 +2,19 @@ require "active_support/duration"
 require "active_support/core_ext/numeric/time"
 
 class Mrsk::Commands::Prune < Mrsk::Commands::Base
-  def images(until_hours: 7.days.in_hours.to_i)
-    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def images
+    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
   end
 
-  def containers(until_hours: 3.days.in_hours.to_i)
-    docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def containers(keep_last: 5)
+    pipe \
+      docker(:ps, "-q", "-a", "--filter", "label=service=#{config.service}", *stopped_containers_filters),
+      "tail -n +#{keep_last + 1}",
+      "while read container_id; do docker rm $container_id; done"
   end
+
+  private
+    def stopped_containers_filters
+      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
+    end
 end

data/lib/mrsk/commands/registry.rb

@@ -2,7 +2,7 @@ class Mrsk::Commands::Registry < Mrsk::Commands::Base
   delegate :registry, to: :config
 
   def login
-    docker :login, registry["server"], "-u", redact(lookup("username")), "-p", redact(lookup("password"))
+    docker :login, registry["server"], "-u", sensitive(lookup("username")), "-p", sensitive(lookup("password"))
   end
 
   def logout

data/lib/mrsk/commands/traefik.rb

@@ -1,7 +1,7 @@
 class Mrsk::Commands::Traefik < Mrsk::Commands::Base
-  delegate :optionize, to: Mrsk::Utils
+  delegate :argumentize, :optionize, to: Mrsk::Utils
 
-  IMAGE = "traefik:v2.9.9"
+  DEFAULT_IMAGE = "traefik:v2.9"
   CONTAINER_PORT = 80
 
   def run
@@ -11,8 +11,9 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
       "--publish", port,
       "--volume", "/var/run/docker.sock:/var/run/docker.sock",
       *config.logging_args,
+      *label_args,
       *docker_options_args,
-      IMAGE,
+      image,
       "--providers.docker",
       "--log.level=DEBUG",
       *cmd_option_args
@@ -56,6 +57,18 @@ class Mrsk::Commands::Traefik < Mrsk::Commands::Base
   end
 
   private
+    def label_args
+      argumentize "--label", labels
+    end
+
+    def labels
+      config.traefik["labels"] || []
+    end
+
+    def image
+      config.traefik.fetch("image") { DEFAULT_IMAGE }
+    end
+
     def docker_options_args
       optionize(config.traefik["options"] || {})
     end

data/lib/mrsk/configuration/boot.rb

@@ -0,0 +1,20 @@
+class Mrsk::Configuration::Boot
+  def initialize(config:)
+    @options = config.raw_config.boot || {}
+    @host_count = config.all_hosts.count
+  end
+
+  def limit
+    limit = @options["limit"]
+
+    if limit.to_s.end_with?("%")
+      @host_count * limit.to_i / 100
+    else
+      limit
+    end
+  end
+
+  def wait
+    @options["wait"]
+  end
+end

data/lib/mrsk/configuration/role.rb

@@ -35,6 +35,21 @@ class Mrsk::Configuration::Role
     argumentize_env_with_secrets env
   end
 
+  def health_check_args
+    if health_check_cmd.present?
+      optionize({ "health-cmd" => health_check_cmd, "health-interval" => "1s" })
+    else
+      []
+    end
+  end
+
+  def health_check_cmd
+    options = specializations["healthcheck"] || {}
+    options = config.healthcheck.merge(options) if running_traefik?
+
+    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  end
+
   def cmd
     specializations["cmd"]
   end
@@ -74,18 +89,23 @@ class Mrsk::Configuration::Role
     def traefik_labels
       if running_traefik?
         {
-          "traefik.http.routers.#{config.service}.rule" => "PathPrefix(`/`)",
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.path" => config.healthcheck["path"],
-          "traefik.http.services.#{config.service}.loadbalancer.healthcheck.interval" => "1s",
-          "traefik.http.middlewares.#{config.service}-retry.retry.attempts" => "5",
-          "traefik.http.middlewares.#{config.service}-retry.retry.initialinterval" => "500ms",
-          "traefik.http.routers.#{config.service}.middlewares" => "#{config.service}-retry@docker"
+          # Setting a service property ensures that the generated service name will be consistent between versions
+          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
+
+          "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
+          "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
+          "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
         }
       else
         {}
       end
     end
 
+    def traefik_service
+      [ config.service, name, config.destination ].compact.join("-")
+    end
+
     def custom_labels
       Hash.new.tap do |labels|
         labels.merge!(config.labels) if config.labels.present?
@@ -121,4 +141,8 @@ class Mrsk::Configuration::Role
         new_env["clear"] = (clear_app_env + clear_role_env).uniq
       end
     end
+
+    def http_health_check(port:, path:)
+      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
+    end
 end