dash 2.12.0 → 3.0.0

data/lib/kamal/cli/doctor/host_checks.rb

@@ -0,0 +1,178 @@
+# Remote readiness checks for a single host, run inside an SSHKit backend.
+#
+# Every check rescues StandardError at its boundary: the doctor's contract is
+# to report broken environments, not crash on them, and a failed connection can
+# raise anything from Net::SSH errors to Errno and DNS resolution errors.
+class Kamal::Cli::Doctor::HostChecks
+  attr_reader :host, :sshkit, :proxy_host
+  delegate :execute, :capture_with_info, to: :sshkit
+
+  def initialize(host, sshkit, proxy_host:)
+    @host = host
+    @sshkit = sshkit
+    @proxy_host = proxy_host
+  end
+
+  def run
+    checks = { ssh: ssh_check }
+
+    if checks[:ssh].ok?
+      checks[:docker] = docker_check
+      checks[:registry] = registry_check
+      checks.merge!(proxy_checks) if proxy_host
+    end
+
+    checks
+  end
+
+  private
+    def result(check, status, detail)
+      Kamal::Cli::Doctor::Result.new(check, host, status, detail)
+    end
+
+    def ssh_check
+      execute "true"
+      result :ssh, :ok, "connected"
+    rescue StandardError => e
+      result :ssh, :fail, "#{e.class}: #{e.message}"
+    end
+
+    def docker_check
+      if execute(*KAMAL.docker.running?, raise_on_non_zero_exit: false)
+        result :docker, :ok, "docker is installed and running"
+      else
+        result :docker, :fail, "docker is not installed or not running (run `kamal server bootstrap`)"
+      end
+    rescue StandardError => e
+      result :docker, :fail, "#{e.class}: #{e.message}"
+    end
+
+    def registry_check
+      return result(:registry, :ok, "local registry, no login required") if KAMAL.registry.local?
+
+      if execute(*KAMAL.registry.login, raise_on_non_zero_exit: false)
+        result :registry, :ok, "logged in to #{registry_name}"
+      else
+        result :registry, :fail, "docker login to #{registry_name} failed (check registry credentials)"
+      end
+    rescue StandardError => e
+      result :registry, :fail, "#{e.class}: #{e.message}"
+    end
+
+    def proxy_checks
+      version, version_error = capture_proxy_version
+
+      {
+        proxy_image: proxy_image_check,
+        proxy_version: proxy_version_check(version, version_error),
+        proxy_socket: proxy_socket_check(proxy_running: version.present?),
+        ports: ports_check(proxy_running: version.present?)
+      }
+    end
+
+    def capture_proxy_version
+      [ capture_with_info(*KAMAL.proxy(host).version).strip.presence, nil ]
+    rescue StandardError => e
+      [ nil, e ]
+    end
+
+    def proxy_image_check
+      image = expected_proxy_image
+
+      if execute(*KAMAL.docker.manifest_available?(image), raise_on_non_zero_exit: false)
+        result :proxy_image, :ok, "#{image} manifest is fetchable"
+      else
+        result :proxy_image, :fail, "cannot fetch the manifest for #{image} (registry unreachable or unauthorized)"
+      end
+    rescue StandardError => e
+      result :proxy_image, :fail, "#{e.class}: #{e.message}"
+    end
+
+    def proxy_version_check(version, error)
+      minimum = Kamal::Configuration::Proxy::Run::MINIMUM_VERSION
+
+      if error
+        result :proxy_version, :warn, "could not determine the running version (#{error.message})"
+      elsif version.nil?
+        result :proxy_version, :ok, "not running (will be started on deploy)"
+      elsif Kamal::Utils.older_version?(version, minimum)
+        result :proxy_version, :fail, "#{version} is older than the minimum #{minimum}, run `kamal proxy reboot` to update"
+      else
+        result :proxy_version, :ok, "#{version} (minimum #{minimum})"
+      end
+    rescue ArgumentError
+      result :proxy_version, :warn, "running image tag #{version} is not a version number"
+    end
+
+    # What a container path has to look like to count as a container runtime
+    # socket when the config no longer names one.
+    DOCKER_SOCKET_PATTERN = %r{docker\.sock\z}
+
+    # The config-time sleep/docker_socket pairing check covers the *current*
+    # config; the running container keeps whatever it was booted with. A proxy
+    # from before the socket was added silently lacks the mount - the failure
+    # mode is one hung request when a sleeping service never wakes - and one
+    # from before it was removed keeps root-equivalent host access.
+    def proxy_socket_check(proxy_running:)
+      expected = KAMAL.config.proxy_run(host)&.docker_socket
+
+      unless proxy_running
+        detail = expected ? "docker socket #{expected} will be mounted on boot" : "no docker socket configured"
+        return result(:proxy_socket, :ok, detail)
+      end
+
+      mounted = capture_with_info(*KAMAL.proxy(host).mount_destinations, raise_on_non_zero_exit: false).split("\n").map(&:strip)
+
+      if expected
+        if mounted.include?(expected)
+          result :proxy_socket, :ok, "docker socket #{expected} is mounted"
+        elsif sleep_configured?
+          result :proxy_socket, :fail, "the running kamal-proxy has no #{expected} mount, so sleeping services never wake - run `kamal proxy reboot`"
+        else
+          # Nothing sleeps yet, so nothing hangs - drift rather than breakage.
+          result :proxy_socket, :warn, "the running kamal-proxy has no #{expected} mount - run `kamal proxy reboot` to apply the current configuration"
+        end
+      elsif (stray = mounted.grep(DOCKER_SOCKET_PATTERN).first)
+        result :proxy_socket, :warn, "the running kamal-proxy mounts #{stray} but the config no longer asks for it - " \
+          "the socket is root-equivalent host access; `kamal proxy reboot` removes it"
+      else
+        result :proxy_socket, :ok, "no docker socket configured or mounted"
+      end
+    rescue StandardError => e
+      result :proxy_socket, :warn, "could not check the docker socket (#{e.message})"
+    end
+
+    def sleep_configured?
+      KAMAL.config.roles.any? { |role| role.running_proxy? && role.proxy.proxy_config["sleep"].present? }
+    end
+
+    def ports_check(proxy_running:)
+      run_config = KAMAL.config.proxy_run(host)
+      return result(:ports, :ok, "proxy ports are not published, nothing to check") if run_config && !run_config.publish?
+
+      http_port = run_config&.http_port || Kamal::Configuration::Proxy::Run::DEFAULT_HTTP_PORT
+      https_port = run_config&.https_port || Kamal::Configuration::Proxy::Run::DEFAULT_HTTPS_PORT
+
+      if proxy_running
+        result :ports, :ok, "ports #{http_port}/#{https_port} held by the running kamal-proxy"
+      elsif (busy = busy_ports(http_port, https_port)).any?
+        result :ports, :fail, "port(s) #{busy.join(", ")} already in use by another process"
+      else
+        result :ports, :ok, "ports #{http_port}/#{https_port} free"
+      end
+    rescue StandardError => e
+      result :ports, :warn, "could not check ports (#{e.message})"
+    end
+
+    def busy_ports(*ports)
+      ports.select { |port| capture_with_info(*KAMAL.server.listeners_on(port), raise_on_non_zero_exit: false).present? }
+    end
+
+    def expected_proxy_image
+      KAMAL.config.proxy_run(host)&.image || "#{KAMAL.config.proxy_boot.image_default}:#{Kamal::Configuration::Proxy::Run::MINIMUM_VERSION}"
+    end
+
+    def registry_name
+      KAMAL.config.registry.server || "Docker Hub"
+    end
+end

data/lib/kamal/cli/doctor.rb

@@ -0,0 +1,112 @@
+require "kamal/sshkit_with_ext"
+
+# Runs read-only deploy readiness checks and collects the results, without ever
+# raising on a broken environment - failures become failing results instead.
+class Kamal::Cli::Doctor
+  include SSHKit::DSL
+
+  HOST_CHECKS = %i[ ssh docker registry proxy_image proxy_version proxy_socket ports ]
+
+  CHECK_TITLES = {
+    ssh: "SSH",
+    docker: "Docker",
+    registry: "Registry",
+    proxy_image: "Proxy image",
+    proxy_version: "Proxy version",
+    proxy_socket: "Proxy docker socket",
+    ports: "Ports",
+    dns: "DNS",
+    certificate: "Certificates",
+    readiness: "Readiness"
+  }.freeze
+
+  STATUS_COLORS = { ok: :green, warn: :yellow, fail: :red }.freeze
+
+  Result = Struct.new(:check, :target, :status, :detail) do
+    def ok?
+      status == :ok
+    end
+
+    def warn?
+      status == :warn
+    end
+
+    def fail?
+      status == :fail
+    end
+
+    def title
+      CHECK_TITLES[check]
+    end
+
+    def to_s
+      "#{status.to_s.upcase} #{target}: #{detail}"
+    end
+  end
+
+  attr_reader :results
+
+  def initialize
+    @results = []
+  end
+
+  def run
+    @results = host_check_results + endpoint_check_results + config_check_results
+  end
+
+  def failures
+    results.select(&:fail?)
+  end
+
+  def warnings
+    results.select(&:warn?)
+  end
+
+  def successful?
+    failures.none?
+  end
+
+  private
+    def host_check_results
+      results_by_host = collect_host_checks
+
+      HOST_CHECKS.flat_map do |check|
+        KAMAL.hosts.filter_map { |host| results_by_host.dig(host, check) }
+      end
+    end
+
+    def collect_host_checks
+      results_by_host = {}
+      mutex = Mutex.new
+      proxy_hosts = KAMAL.proxy_hosts
+      error = nil
+
+      begin
+        on(KAMAL.hosts) do |host|
+          checks = Kamal::Cli::Doctor::HostChecks.new(host.hostname, self, proxy_host: proxy_hosts.include?(host.hostname)).run
+          mutex.synchronize { results_by_host[host.hostname] = checks }
+        end
+      # Only ExecuteError: sshkit 1.25 has no MultipleExecuteError, and naming
+      # a constant that does not exist turns "a host's checks failed" into a
+      # NameError - the opposite of the doctor's never-crash contract.
+      rescue SSHKit::Runner::ExecuteError => e
+        # Per-check errors are captured inside HostChecks; getting here means a
+        # host's checks never completed at all. Record those as SSH failures below.
+        error = e
+      end
+
+      KAMAL.hosts.each do |host|
+        results_by_host[host] ||= { ssh: Result.new(:ssh, host, :fail, "could not run checks (#{error&.message || "connection failed"})") }
+      end
+
+      results_by_host
+    end
+
+    def endpoint_check_results
+      Kamal::Cli::Doctor::EndpointChecks.new.run
+    end
+
+    def config_check_results
+      Kamal::Cli::Doctor::ConfigChecks.new.run
+    end
+end

data/lib/kamal/cli/healthcheck/drift_error.rb

@@ -0,0 +1,7 @@
+# Raised when the config declares a healthcheck the running container does not have.
+#
+# Deliberately NOT a Kamal::Cli::Healthcheck::Error: the poller retries that class until
+# deploy_timeout, and drift is deterministic — `.State.Health` exists from the moment a
+# container with a healthcheck is created, so retrying only burns the timeout before
+# reporting the same thing.
+class Kamal::Cli::Healthcheck::DriftError < StandardError; end

data/lib/kamal/cli/healthcheck/poller.rb

@@ -1,30 +1,41 @@
 module Kamal::Cli::Healthcheck::Poller
   extend self
 
-  def wait_for_healthy(&block)
+  NO_HEALTHCHECK = Kamal::Commands::Base::NO_HEALTHCHECK
+
+  def wait_for_healthy(role:, &block)
     attempt = 1
     timeout_at = Time.now + KAMAL.config.deploy_timeout
-    readiness_delay = KAMAL.config.readiness_delay
+    readiness_delay = role.readiness_delay
 
     begin
       status = block.call
 
-      if status == "running"
-        # Wait for the readiness delay and confirm it is still running
-        if readiness_delay > 0
-          info "Container is running, waiting for readiness delay of #{readiness_delay} seconds"
-          sleep readiness_delay
-          status = block.call
+      if unchecked?(status)
+        ensure_no_healthcheck_drift(role, status)
+
+        if docker_state(status) == "running"
+          announce_missing_gate(role, readiness_delay)
+
+          # Wait for the readiness delay and confirm it is still running
+          if readiness_delay > 0
+            sleep readiness_delay
+            status = block.call
+            ensure_no_healthcheck_drift(role, status)
+          end
         end
       end
 
-      unless %w[ running healthy ].include?(status)
+      unless acceptable?(status)
         raise Kamal::Cli::Healthcheck::Error, "container not ready after #{KAMAL.config.deploy_timeout} seconds (#{status})"
       end
     rescue Kamal::Cli::Healthcheck::Error => e
       time_left = timeout_at - Time.now
       if time_left > 0
-        sleep [ attempt, time_left ].min
+        sleep_for = [ attempt, time_left ].min
+        elapsed = KAMAL.config.deploy_timeout - time_left
+        info "Container not ready yet, retrying in #{sleep_for.ceil}s (#{elapsed.round}s elapsed, #{time_left.round}s left)"
+        sleep sleep_for
         attempt += 1
         retry
       else
@@ -36,7 +47,46 @@ module Kamal::Cli::Healthcheck::Poller
   end
 
   private
+    def unchecked?(status)
+      status.to_s.start_with?("#{NO_HEALTHCHECK}:")
+    end
+
+    def docker_state(status)
+      status.to_s.delete_prefix("#{NO_HEALTHCHECK}:")
+    end
+
+    def acceptable?(status)
+      status == "healthy" || (unchecked?(status) && docker_state(status) == "running")
+    end
+
+    # The config asked docker to probe this container and docker is not probing it — the flags
+    # never reached `docker run`. Accepting it would let the deploy pass without ever checking
+    # readiness, which is exactly the failure the healthcheck was added to prevent.
+    def ensure_no_healthcheck_drift(role, status)
+      return unless %i[ healthcheck docker_options ].include?(role.readiness_source)
+
+      raise Kamal::Cli::Healthcheck::DriftError,
+        "#{role.name} declares a healthcheck but the container reports none (#{status}) — the flags never reached docker. " \
+        "Compare `servers/#{role.name}` in your deploy config against `docker inspect --format '{{json .Config.Healthcheck}}'` " \
+        "on the container; the deploy would otherwise be accepted without ever probing it."
+    end
+
+    # A role nobody made a readiness decision for still boots — the config-time warning owns that
+    # policy — but it must not do so silently, because the delay is the only gate there is.
+    def announce_missing_gate(role, readiness_delay)
+      if role.readiness_gated?
+        info "Container is running, waiting for readiness delay of #{readiness_delay} seconds"
+      else
+        warn "#{role.name} has no healthcheck — accepting the container as ready after the #{readiness_delay}s readiness delay alone. " \
+          "Add a `healthcheck:` block to gate the deploy on readiness, or `healthcheck: false` to accept this explicitly."
+      end
+    end
+
     def info(message)
       SSHKit.config.output.info(message)
     end
+
+    def warn(message)
+      SSHKit.config.output.warn(message)
+    end
 end

data/lib/kamal/cli/main.rb

@@ -23,6 +23,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       runtime = print_runtime do
         invoke_options = deploy_options
 
+        print_config_banner
+
+        say "Validate configuration and secrets...", :magenta
+        KAMAL.config.validate_secrets!(include_accessories: boot_accessories)
+
         if options[:skip_push]
           say "Pull app image...", :magenta
           invoke "kamal:cli:build:pull", [], invoke_options
@@ -66,6 +71,11 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       runtime = print_runtime do
         invoke_options = deploy_options
 
+        print_config_banner
+
+        say "Validate configuration and secrets...", :magenta
+        KAMAL.config.validate_secrets!
+
         if options[:skip_push]
           say "Pull app image...", :magenta
           invoke "kamal:cli:build:pull", [], invoke_options
@@ -153,6 +163,23 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     puts "No documentation found for #{section}"
   end
 
+  desc "doctor", "Diagnose deploy readiness of servers, registry, proxy, ports, DNS, certificates, and per-role readiness gates"
+  def doctor
+    say "Running readiness checks...", :magenta
+    pre_connect_if_required
+
+    doctor = Kamal::Cli::Doctor.new
+    doctor.run
+
+    print_doctor_report doctor.results
+
+    if doctor.successful?
+      say doctor_summary(doctor), :green
+    else
+      raise Kamal::Cli::DoctorError, doctor_failure_message(doctor)
+    end
+  end
+
   desc "init", "Create config stub in config/deploy.yml and secrets stub in .kamal"
   option :bundle, type: :boolean, default: false, desc: "Add Kamal to the Gemfile and create a bin/kamal binstub"
   def init
@@ -296,4 +323,40 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       base_options = base_options.except("no_cache") unless base_options["no_cache"]
       { "version" => KAMAL.config.version }.merge(base_options)
     end
+
+    def print_doctor_report(results)
+      results.group_by(&:title).each do |title, rows|
+        say title
+        rows.each { |row| say "  #{row}", Kamal::Cli::Doctor::STATUS_COLORS[row.status] }
+      end
+    end
+
+    def doctor_summary(doctor)
+      if doctor.warnings.any?
+        "Looks ready to deploy, with #{doctor.warnings.count} warning(s) to review"
+      else
+        "Everything looks ready to deploy"
+      end
+    end
+
+    def doctor_failure_message(doctor)
+      failing = doctor.failures.map { |failure| "  #{failure.title} - #{failure}" }.join("\n")
+      "Found #{doctor.failures.count} failing check(s):\n#{failing}"
+    end
+
+    def print_config_banner
+      config = KAMAL.config
+
+      say "Deploying #{config.service}#{" to #{config.destination}" if config.destination} (version #{config.abbreviated_version})", :magenta
+      config.roles.each do |role|
+        hosts = "#{role.hosts.count} #{"host".pluralize(role.hosts.count)} (#{role.hosts.join(", ")})"
+        say "  #{role.name}: #{hosts} — readiness: #{role.readiness_description}", (:yellow if role.readiness_source == :none)
+      end
+      say "  proxy: #{config.proxy_hosts.join(", ")}" if config.proxy_hosts.any?
+      if config.proxy.load_balancing?
+        reason = " (auto-enabled: primary role #{config.primary_role.name} has #{config.primary_role.hosts.count} hosts)" unless config.proxy.loadbalancer.present?
+        say "  loadbalancer: #{config.proxy.effective_loadbalancer}#{reason}"
+      end
+      say "  timeouts: deploy #{config.deploy_timeout}s, drain #{config.drain_timeout}s, readiness delay #{config.readiness_delay}s"
+    end
 end

data/lib/kamal/cli/proxy/drift.rb

@@ -0,0 +1,39 @@
+class Kamal::Cli::Proxy::Drift
+  attr_reader :host, :sshkit
+  delegate :capture_with_info, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def container_exists?
+    capture_with_info(*proxy.container_id, raise_on_non_zero_exit: false).strip.present?
+  end
+
+  # A proxy container has drifted when it was started with a different config
+  # digest than the one the current configuration produces. Containers booted
+  # by older kamal versions carry no digest label and count as drifted, so
+  # they converge on the first deploy after upgrading.
+  def drifted?
+    return @drifted if defined?(@drifted)
+    @drifted = container_exists? && current_digest != expected_digest
+  end
+
+  def expected_digest
+    @expected_digest ||= if proxy.proxy_run_config
+      proxy.proxy_run_config.config_digest
+    else
+      Kamal::Configuration::Proxy::Run.digest(capture_with_info(*proxy.boot_config).strip)
+    end
+  end
+
+  private
+    def current_digest
+      capture_with_info(*proxy.config_digest, raise_on_non_zero_exit: false).strip
+    end
+
+    def proxy
+      @proxy ||= KAMAL.proxy(host)
+    end
+end

data/lib/kamal/cli/proxy/loadbalancer_claim.rb

@@ -0,0 +1,70 @@
+# Ownership bookkeeping for a load balancer host that more than one kamal app
+# may be deploying to. Kamal only ever sees one deploy.yml at a time, so the
+# other apps are visible solely through the files they left on the host.
+class Kamal::Cli::Proxy::LoadbalancerClaim
+  attr_reader :host, :sshkit
+  delegate :capture_with_info, :execute, :upload!, :warn, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  # The load balancer registers services under the bare service name, so a
+  # second app - or a second destination of this app - deploying the same name
+  # would silently take over its routes. Claim the name on first deploy and fail
+  # loudly rather than quietly stealing it.
+  def claim_service
+    owner = capture_with_info(*commands.read_service_owner, raise_on_non_zero_exit: false).strip
+
+    if owner.present? && owner != config.owner_token
+      raise "Service '#{config.config.service}' is already registered on the load balancer at #{host} by #{repository_in(owner)}. " \
+            "Deploying would take over its routes - rename this app's service or use a different load balancer."
+    end
+
+    execute *commands.ensure_services_directory
+    upload! StringIO.new(config.owner_token), config.service_owner_file
+  end
+
+  # Every app sharing a load balancer boots the same container, so their
+  # proxy/run configurations have to agree. A mismatch owned by another app is
+  # an error; a mismatch owned by this app is ordinary drift, reported the same
+  # way the per-host proxy reports it and fixed by `kamal proxy reboot`.
+  def claim_run_config(replace: false)
+    record = capture_with_info(*commands.read_run_config_record, raise_on_non_zero_exit: false).strip
+
+    if record.present?
+      *owner_parts, digest = record.split(" ")
+      owner = owner_parts.join(" ")
+
+      if digest != config.run_config_digest
+        if owner != config.owner_token
+          raise "The load balancer on #{host} was booted by #{repository_in(owner)} with a different proxy/run configuration. " \
+                "Apps sharing a load balancer must agree on proxy/run - align the configurations or use a different load balancer host."
+        elsif !replace
+          warn "The load balancer on #{host} is running with a configuration that no longer matches the deploy config - " \
+               "run `kamal proxy reboot` to apply the new configuration."
+          return
+        end
+      end
+    end
+
+    execute *commands.ensure_directory
+    upload! StringIO.new(config.run_config_record), config.run_config_file
+  end
+
+  private
+    def config
+      KAMAL.loadbalancer_config
+    end
+
+    def commands
+      KAMAL.loadbalancer
+    end
+
+    # Owner tokens are "<service-and-destination> <repository>"; the repository
+    # is the half an operator recognises as "the other app".
+    def repository_in(owner)
+      owner.split(" ").last
+    end
+end

data/lib/kamal/cli/proxy/loadbalancer_reboot.rb

@@ -0,0 +1,41 @@
+# Replaces the loadbalancer container with one booted from the current
+# configuration. Shared by `kamal proxy reboot` and the drift-detected reboot
+# on boot - the same sequence either way, so the two paths cannot diverge.
+#
+# Unlike the proxy hosts' port-holder handoff this is stop -> run: the
+# loadbalancer keeps its service state in the config volume, which the
+# replacement container re-mounts, so every app's routes survive the gap.
+class Kamal::Cli::Proxy::LoadbalancerReboot
+  attr_reader :host, :sshkit
+  delegate :execute, :capture_with_info, :info, :upload!, to: :sshkit
+
+  def initialize(host, sshkit)
+    @host = host
+    @sshkit = sshkit
+  end
+
+  def run
+    execute *KAMAL.auditor.record("Rebooted loadbalancer"), verbosity: :debug
+    execute *KAMAL.registry.login
+
+    info "Stopping and removing #{KAMAL.loadbalancer.container_name} on #{host}, if running..."
+    execute *KAMAL.loadbalancer.stop, raise_on_non_zero_exit: false
+    execute *KAMAL.loadbalancer.remove_container
+
+    if (lb_run = KAMAL.loadbalancer_config.run).secrets?
+      execute *KAMAL.loadbalancer.ensure_proxy_directory
+      upload! lb_run.secrets_io, lb_run.secrets_path, mode: "0600"
+    else
+      execute *KAMAL.loadbalancer.remove_proxy_secrets_file, raise_on_non_zero_exit: false
+    end
+
+    execute *KAMAL.loadbalancer.ensure_apps_config_directory
+    Kamal::Cli::Proxy::LoadbalancerClaim.new(host, sshkit).claim_run_config(replace: true)
+    execute *KAMAL.loadbalancer.run
+
+    # kamal-proxy keeps its service state in the config volume, which the
+    # replacement container re-mounts - every app's routes survive.
+    services = capture_with_info(*KAMAL.loadbalancer.list).strip
+    info "Services registered on the load balancer at #{host} after reboot:\n#{services}"
+  end
+end