mrsk 0.11.0 → 0.12.1

data/lib/mrsk/commander.rb

@@ -2,11 +2,12 @@ require "active_support/core_ext/enumerable"
 require "active_support/core_ext/module/delegation"
 
 class Mrsk::Commander
-  attr_accessor :verbosity, :lock_count
+  attr_accessor :verbosity, :holding_lock, :hold_lock_on_error
 
   def initialize
     self.verbosity = :info
-    self.lock_count = 0
+    self.holding_lock = false
+    self.hold_lock_on_error = false
   end
 
   def config
@@ -35,7 +36,7 @@ class Mrsk::Commander
   end
 
   def primary_host
-    specific_hosts&.first || config.primary_web_host
+    specific_hosts&.first || specific_roles&.first&.primary_host || config.primary_web_host
   end
 
   def roles
@@ -50,6 +51,14 @@ class Mrsk::Commander
     end
   end
 
+  def boot_strategy
+    if config.boot.limit.present?
+      { in: :groups, limit: config.boot.limit, wait: config.boot.wait }
+    else
+      {}
+    end
+  end
+
   def roles_on(host)
     roles.select { |role| role.hosts.include?(host.to_s) }.map(&:name)
   end
@@ -75,14 +84,18 @@ class Mrsk::Commander
     Mrsk::Commands::Accessory.new(config, name: name)
   end
 
-  def auditor(role: nil)
-    Mrsk::Commands::Auditor.new(config, role: role)
+  def auditor(**details)
+    Mrsk::Commands::Auditor.new(config, **details)
   end
 
   def builder
     @builder ||= Mrsk::Commands::Builder.new(config)
   end
 
+  def docker
+    @docker ||= Mrsk::Commands::Docker.new(config)
+  end
+
   def healthcheck
     @healthcheck ||= Mrsk::Commands::Healthcheck.new(config)
   end
@@ -115,6 +128,14 @@ class Mrsk::Commander
     SSHKit.config.output_verbosity = old_level
   end
 
+  def holding_lock?
+    self.holding_lock
+  end
+
+  def hold_lock_on_error?
+    self.hold_lock_on_error
+  end
+
   private
     # Lazy setup of SSHKit
     def configure_sshkit_with(config)

data/lib/mrsk/commands/app.rb

@@ -15,6 +15,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
       "--name", container_name,
       "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
       *role.env_args,
+      *role.health_check_args,
       *config.logging_args,
       *config.volume_args,
       *role.label_args,
@@ -27,9 +28,13 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :start, container_name
   end
 
+  def status(version:)
+    pipe container_id_for_version(version), xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
+  end
+
   def stop(version: nil)
     pipe \
-      version ? container_id_for_version(version) : current_container_id,
+      version ? container_id_for_version(version) : current_running_container_id,
       xargs(config.stop_wait_time ? docker(:stop, "-t", config.stop_wait_time) : docker(:stop))
   end
 
@@ -40,7 +45,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
 
   def logs(since: nil, lines: nil, grep: nil)
     pipe \
-      current_container_id,
+      current_running_container_id,
       "xargs docker logs#{" --since #{since}" if since}#{" --tail #{lines}" if lines} 2>&1",
       ("grep '#{grep}'" if grep)
   end
@@ -48,7 +53,7 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   def follow_logs(host:, grep: nil)
     run_over_ssh \
       pipe(
-        current_container_id,
+        current_running_container_id,
         "xargs docker logs --timestamps --tail 10 --follow 2>&1",
         (%(grep "#{grep}") if grep)
       ),
@@ -82,8 +87,8 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
 
-  def current_container_id
-    docker :ps, "--quiet", *filter_args
+  def current_running_container_id
+    docker :ps, "--quiet", *filter_args(status: :running), "--latest"
   end
 
   def container_id_for_version(version)
@@ -91,11 +96,14 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
   end
 
   def current_running_version
-    # FIXME: Find more graceful way to extract the version from "app-version" than using sed and tail!
+    list_versions("--latest", status: :running)
+  end
+
+  def list_versions(*docker_args, status: nil)
     pipe \
-      docker(:ps, *filter_args, "--format", '"{{.Names}}"'),
-      %(sed 's/-/\\n/g'),
-      "tail -n 1"
+      docker(:ps, *filter_args(status: status), *docker_args, "--format", '"{{.Names}}"'),
+      %(grep -oE "\\-[^-]+$"), # Extract SHA from "service-role-dest-SHA"
+      %(cut -c 2-)
   end
 
   def list_containers
@@ -128,20 +136,25 @@ class Mrsk::Commands::App < Mrsk::Commands::Base
     docker :image, :prune, "--all", "--force", *filter_args
   end
 
+  def tag_current_as_latest
+    docker :tag, config.absolute_image, config.latest_image
+  end
+
 
   private
     def container_name(version = nil)
       [ config.service, role, config.destination, version || config.version ].compact.join("-")
     end
 
-    def filter_args
-      argumentize "--filter", filters
+    def filter_args(status: nil)
+      argumentize "--filter", filters(status: status)
     end
 
-    def filters
+    def filters(status: nil)
       [ "label=service=#{config.service}" ].tap do |filters|
         filters << "label=destination=#{config.destination}" if config.destination
         filters << "label=role=#{role}" if role
+        filters << "status=#{status}" if status
       end
     end
 end

data/lib/mrsk/commands/auditor.rb

@@ -1,24 +1,24 @@
-require "active_support/core_ext/time/conversions"
+require "time"
 
 class Mrsk::Commands::Auditor < Mrsk::Commands::Base
-  attr_reader :role
+  attr_reader :details
 
-  def initialize(config, role: nil)
+  def initialize(config, **details)
     super(config)
-    @role = role
+    @details = default_details.merge(details)
   end
 
   # Runs remotely
-  def record(line)
+  def record(line, **details)
     append \
-      [ :echo, tagged_record_line(line) ],
+      [ :echo, *audit_tags(**details), line ],
       audit_log_file
   end
 
   # Runs locally
-  def broadcast(line)
+  def broadcast(line, **details)
     if broadcast_cmd = config.audit_broadcast_cmd
-      [ broadcast_cmd, tagged_broadcast_line(line) ]
+      [ broadcast_cmd, *broadcast_args(line, **details), env: env_for(event: line, **details) ]
     end
   end
 
@@ -31,27 +31,29 @@ class Mrsk::Commands::Auditor < Mrsk::Commands::Base
       [ "mrsk", config.service, config.destination, "audit.log" ].compact.join("-")
     end
 
-    def tagged_record_line(line)
-      tagged_line recorded_at_tag, performer_tag, role_tag, line
+    def default_details
+      { recorded_at: Time.now.utc.iso8601,
+        performer: `whoami`.chomp,
+        destination: config.destination }
     end
 
-    def tagged_broadcast_line(line)
-      tagged_line performer_tag, role_tag, line
+    def audit_tags(**details)
+      tags_for **self.details.merge(details)
     end
 
-    def tagged_line(*tags_and_line)
-      "'#{tags_and_line.compact.join(" ")}'"
+    def broadcast_args(line, **details)
+      "'#{broadcast_tags(**details).join(" ")} #{line}'"
     end
 
-    def recorded_at_tag
-      "[#{Time.now.to_fs(:db)}]"
+    def broadcast_tags(**details)
+      tags_for **self.details.merge(details).except(:recorded_at)
     end
 
-    def performer_tag
-      "[#{`whoami`.strip}]"
+    def tags_for(**details)
+      details.compact.values.map { |value| "[#{value}]" }
     end
 
-    def role_tag
-      "[#{role}]" if role
+    def env_for(**details)
+      self.details.merge(details).compact.transform_keys { |detail| "MRSK_#{detail.upcase}" }
     end
 end

data/lib/mrsk/commands/base.rb

@@ -2,6 +2,9 @@ module Mrsk::Commands
   class Base
     delegate :sensitive, :argumentize, to: Mrsk::Utils
 
+    DOCKER_HEALTH_STATUS_FORMAT = "'{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}'"
+    DOCKER_HEALTH_LOG_FORMAT    = "'{{json .State.Health}}'"
+
     attr_accessor :config
 
     def initialize(config)

data/lib/mrsk/commands/builder/base.rb

@@ -1,4 +1,7 @@
+
 class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
+  class BuilderError < StandardError; end
+
   delegate :argumentize, to: Mrsk::Utils
 
   def clean
@@ -7,7 +10,6 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
 
   def pull
     docker :pull, config.absolute_image
-    docker :pull, config.latest_image
   end
 
   def build_options
@@ -18,6 +20,7 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
     context
   end
 
+
   private
     def build_tags
       [ "-t", config.absolute_image, "-t", config.latest_image ]
@@ -36,7 +39,11 @@ class Mrsk::Commands::Builder::Base < Mrsk::Commands::Base
     end
 
     def build_dockerfile
-      argumentize "--file", dockerfile
+      if Pathname.new(File.expand_path(dockerfile)).exist?
+        argumentize "--file", dockerfile
+      else
+        raise BuilderError, "Missing #{dockerfile}"
+      end
     end
 
     def args

data/lib/mrsk/commands/builder.rb

@@ -2,7 +2,7 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
   delegate :create, :remove, :push, :clean, :pull, :info, to: :target
 
   def name
-    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore
+    target.class.to_s.remove("Mrsk::Commands::Builder::").underscore.inquiry
   end
 
   def target
@@ -33,4 +33,24 @@ class Mrsk::Commands::Builder < Mrsk::Commands::Base
   def multiarch_remote
     @multiarch_remote ||= Mrsk::Commands::Builder::Multiarch::Remote.new(config)
   end
+
+
+  def ensure_local_dependencies_installed
+    if name.native?
+      ensure_local_docker_installed
+    else
+      combine \
+        ensure_local_docker_installed,
+        ensure_local_buildx_installed
+    end
+  end
+
+  private
+    def ensure_local_docker_installed
+      docker "--version"
+    end
+
+    def ensure_local_buildx_installed
+      docker :buildx, "version"
+    end
 end

data/lib/mrsk/commands/docker.rb

@@ -0,0 +1,21 @@
+class Mrsk::Commands::Docker < Mrsk::Commands::Base
+  # Install Docker using the https://github.com/docker/docker-install convenience script.
+  def install
+    pipe [ :curl, "-fsSL", "https://get.docker.com" ], :sh
+  end
+
+  # Checks the Docker client version. Fails if Docker is not installed.
+  def installed?
+    docker "-v"
+  end
+
+  # Checks the Docker server version. Fails if Docker is not running.
+  def running?
+    docker :version
+  end
+
+  # Do we have superuser access to install Docker and start system services?
+  def superuser?
+    [ '[ "${EUID:-$(id -u)}" -eq 0 ]' ]
+  end
+end

data/lib/mrsk/commands/healthcheck.rb

@@ -11,14 +11,19 @@ class Mrsk::Commands::Healthcheck < Mrsk::Commands::Base
       "--label", "service=#{container_name}",
       "-e", "MRSK_CONTAINER_NAME=\"#{container_name}\"",
       *web.env_args,
+      *web.health_check_args,
       *config.volume_args,
       *web.option_args,
       config.absolute_image,
       web.cmd
   end
 
-  def curl
-    [ :curl, "--silent", "--output", "/dev/null", "--write-out", "'%{http_code}'", "--max-time", "2", health_url ]
+  def status
+    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_STATUS_FORMAT))
+  end
+
+  def container_health_log
+    pipe container_id, xargs(docker(:inspect, "--format", DOCKER_HEALTH_LOG_FORMAT))
   end
 
   def logs

data/lib/mrsk/commands/lock.rb

@@ -1,5 +1,5 @@
 require "active_support/duration"
-require "active_support/core_ext/numeric/time"
+require "time"
 
 class Mrsk::Commands::Lock < Mrsk::Commands::Base
   def acquire(message, version)
@@ -49,7 +49,7 @@ class Mrsk::Commands::Lock < Mrsk::Commands::Base
 
     def lock_details(message, version)
       <<~DETAILS.strip
-        Locked by: #{locked_by} at #{Time.now.gmtime}
+        Locked by: #{locked_by} at #{Time.now.utc.iso8601}
         Version: #{version}
         Message: #{message}
       DETAILS

data/lib/mrsk/commands/prune.rb

@@ -2,11 +2,19 @@ require "active_support/duration"
 require "active_support/core_ext/numeric/time"
 
 class Mrsk::Commands::Prune < Mrsk::Commands::Base
-  def images(until_hours: 7.days.in_hours.to_i)
-    docker :image, :prune, "--all", "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def images
+    docker :image, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "dangling=true"
   end
 
-  def containers(until_hours: 3.days.in_hours.to_i)
-    docker :container, :prune, "--force", "--filter", "label=service=#{config.service}", "--filter", "until=#{until_hours}h"
+  def containers(keep_last: 5)
+    pipe \
+      docker(:ps, "-q", "-a", "--filter", "label=service=#{config.service}", *stopped_containers_filters),
+      "tail -n +#{keep_last + 1}",
+      "while read container_id; do docker rm $container_id; done"
   end
+
+  private
+    def stopped_containers_filters
+      [ "created", "exited", "dead" ].flat_map { |status| ["--filter", "status=#{status}"] }
+    end
 end

data/lib/mrsk/configuration/boot.rb

@@ -0,0 +1,20 @@
+class Mrsk::Configuration::Boot
+  def initialize(config:)
+    @options = config.raw_config.boot || {}
+    @host_count = config.all_hosts.count
+  end
+
+  def limit
+    limit = @options["limit"]
+
+    if limit.to_s.end_with?("%")
+      @host_count * limit.to_i / 100
+    else
+      limit
+    end
+  end
+
+  def wait
+    @options["wait"]
+  end
+end

data/lib/mrsk/configuration/role.rb

@@ -35,6 +35,21 @@ class Mrsk::Configuration::Role
     argumentize_env_with_secrets env
   end
 
+  def health_check_args
+    if health_check_cmd.present?
+      optionize({ "health-cmd" => health_check_cmd, "health-interval" => "1s" })
+    else
+      []
+    end
+  end
+
+  def health_check_cmd
+    options = specializations["healthcheck"] || {}
+    options = config.healthcheck.merge(options) if running_traefik?
+
+    options["cmd"] || http_health_check(port: options["port"], path: options["path"])
+  end
+
   def cmd
     specializations["cmd"]
   end
@@ -74,9 +89,10 @@ class Mrsk::Configuration::Role
     def traefik_labels
       if running_traefik?
         {
+          # Setting a service property ensures that the generated service name will be consistent between versions
+          "traefik.http.services.#{traefik_service}.loadbalancer.server.scheme" => "http",
+
           "traefik.http.routers.#{traefik_service}.rule" => "PathPrefix(`/`)",
-          "traefik.http.services.#{traefik_service}.loadbalancer.healthcheck.path" => config.healthcheck["path"],
-          "traefik.http.services.#{traefik_service}.loadbalancer.healthcheck.interval" => "1s",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.attempts" => "5",
           "traefik.http.middlewares.#{traefik_service}-retry.retry.initialinterval" => "500ms",
           "traefik.http.routers.#{traefik_service}.middlewares" => "#{traefik_service}-retry@docker"
@@ -125,4 +141,8 @@ class Mrsk::Configuration::Role
         new_env["clear"] = (clear_app_env + clear_role_env).uniq
       end
     end
+
+    def http_health_check(port:, path:)
+      "curl -f #{URI.join("http://localhost:#{port}", path)} || exit 1" if path.present? || port.present?
+    end
 end

data/lib/mrsk/configuration.rb

@@ -87,6 +87,10 @@ class Mrsk::Configuration
     roles.select(&:running_traefik?).flat_map(&:hosts).uniq
   end
 
+  def boot
+    Mrsk::Configuration::Boot.new(config: self)
+  end
+
 
   def repository
     [ raw_config.registry["server"], image ].compact.join("/")

data/lib/mrsk/sshkit_with_ext.rb

@@ -1,12 +1,52 @@
 require "sshkit"
 require "sshkit/dsl"
+require "active_support/core_ext/hash/deep_merge"
+require "json"
 
 class SSHKit::Backend::Abstract
-  def capture_with_info(*args)
-    capture(*args, verbosity: Logger::INFO)
+  def capture_with_info(*args, **kwargs)
+    capture(*args, **kwargs, verbosity: Logger::INFO)
+  end
+
+  def capture_with_pretty_json(*args, **kwargs)
+    JSON.pretty_generate(JSON.parse(capture(*args, **kwargs)))
   end
 
   def puts_by_host(host, output, type: "App")
     puts "#{type} Host: #{host}\n#{output}\n\n"
   end
+
+  # Our execution pattern is for the CLI execute args lists returned
+  # from commands, but this doesn't support returning execution options
+  # from the command.
+  #
+  # Support this by using kwargs for CLI options and merging with the
+  # args-extracted options.
+  module CommandEnvMerge
+    private
+
+    # Override to merge options returned by commands in the args list with
+    # options passed by the CLI and pass them along as kwargs.
+    def command(args, options)
+      more_options, args = args.partition { |a| a.is_a? Hash }
+      more_options << options
+
+      build_command(args, **more_options.reduce(:deep_merge))
+    end
+
+    # Destructure options to pluck out env for merge
+    def build_command(args, env: nil, **options)
+      # Rely on native Ruby kwargs precedence rather than explicit Hash merges
+      SSHKit::Command.new(*args, **default_command_options, **options, env: env_for(env))
+    end
+
+    def default_command_options
+      { in: pwd_path, host: @host, user: @user, group: @group }
+    end
+
+    def env_for(env)
+      @env.to_h.merge(env.to_h)
+    end
+  end
+  prepend CommandEnvMerge
 end

data/lib/mrsk/utils/healthcheck_poller.rb

@@ -0,0 +1,39 @@
+class Mrsk::Utils::HealthcheckPoller
+  TRAEFIK_HEALTHY_DELAY = 2
+
+  class HealthcheckError < StandardError; end
+
+  class << self
+    def wait_for_healthy(pause_after_ready: false, &block)
+      attempt = 1
+      max_attempts = MRSK.config.healthcheck["max_attempts"]
+
+      begin
+        case status = block.call
+        when "healthy"
+          sleep TRAEFIK_HEALTHY_DELAY if pause_after_ready
+        when "running" # No health check configured
+          sleep MRSK.config.readiness_delay if pause_after_ready
+        else
+          raise HealthcheckError, "container not ready (#{status})"
+        end
+      rescue HealthcheckError => e
+        if attempt <= max_attempts
+          info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
+          sleep attempt
+          attempt += 1
+          retry
+        else
+          raise
+        end
+      end
+
+      info "Container is healthy!"
+    end
+
+    private
+      def info(message)
+        SSHKit.config.output.info(message)
+      end
+  end
+end

data/lib/mrsk/utils.rb

@@ -1,6 +1,8 @@
 module Mrsk::Utils
   extend self
 
+  DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX = /\$(?!{[^\}]*\})/
+
   # Return a list of escaped shell arguments using the same named argument against the passed attributes (hash or array).
   def argumentize(argument, attributes, sensitive: false)
     Array(attributes).flat_map do |key, value|
@@ -75,7 +77,9 @@ module Mrsk::Utils
 
   # Escape a value to make it safe for shell use.
   def escape_shell_value(value)
-    value.to_s.dump.gsub(/`/, '\\\\`')
+    value.to_s.dump
+      .gsub(/`/, '\\\\`')
+      .gsub(DOLLAR_SIGN_WITHOUT_SHELL_EXPANSION_REGEX, '\$')
   end
 
   # Abbreviate a git revhash for concise display

data/lib/mrsk/version.rb

@@ -1,3 +1,3 @@
 module Mrsk
-  VERSION = "0.11.0"
+  VERSION = "0.12.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: mrsk
 version: !ruby/object:Gem::Version
-  version: 0.11.0
+  version: 0.12.1
 platform: ruby
 authors:
 - David Heinemeier Hansson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-17 00:00:00.000000000 Z
+date: 2023-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -201,6 +201,7 @@ files:
 - lib/mrsk/commands/builder/multiarch/remote.rb
 - lib/mrsk/commands/builder/native.rb
 - lib/mrsk/commands/builder/native/remote.rb
+- lib/mrsk/commands/docker.rb
 - lib/mrsk/commands/healthcheck.rb
 - lib/mrsk/commands/lock.rb
 - lib/mrsk/commands/prune.rb
@@ -208,9 +209,11 @@ files:
 - lib/mrsk/commands/traefik.rb
 - lib/mrsk/configuration.rb
 - lib/mrsk/configuration/accessory.rb
+- lib/mrsk/configuration/boot.rb
 - lib/mrsk/configuration/role.rb
 - lib/mrsk/sshkit_with_ext.rb
 - lib/mrsk/utils.rb
+- lib/mrsk/utils/healthcheck_poller.rb
 - lib/mrsk/utils/sensitive.rb
 - lib/mrsk/version.rb
 homepage: https://github.com/rails/mrsk