kamal 1.8.3 → 2.7.0

data/lib/kamal/cli/build.rb

@@ -5,15 +5,22 @@ class Kamal::Cli::Build < Kamal::Cli::Base
 
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    push
-    pull
+    invoke :push
+    invoke :pull
   end
 
   desc "push", "Build and push app image to registry"
+  option :output, type: :string, default: "registry", banner: "export_type", desc: "Exported type for the build result, and may be any exported type supported by 'buildx --output'."
   def push
     cli = self
 
-    verify_local_dependencies
+    # Ensure pre-connect hooks run before the build, they may needed for a remote builder
+    # or the pre-build hooks.
+    pre_connect_if_required
+
+    ensure_docker_installed
+    login_to_registry_locally
+
     run_hook "pre-build"
 
     uncommitted_changes = Kamal::Git.uncommitted_changes
@@ -30,49 +37,52 @@ class Kamal::Cli::Build < Kamal::Cli::Base
       say "Building with uncommitted changes:\n #{uncommitted_changes}", :yellow
     end
 
-    # Get the command here to ensure the Dir.chdir doesn't interfere with it
-    push = KAMAL.builder.push
+    with_env(KAMAL.config.builder.secrets) do
+      run_locally do
+        begin
+          execute *KAMAL.builder.inspect_builder
+        rescue SSHKit::Command::Failed => e
+          if e.message =~ /(context not found|no builder|no compatible builder|does not exist)/
+            warn "Missing compatible builder, so creating a new one first"
+            begin
+              cli.remove
+            rescue SSHKit::Command::Failed
+              raise unless e.message =~ /(context not found|no builder|does not exist)/
+            end
+            cli.create
+          else
+            raise
+          end
+        end
 
-    run_locally do
-      begin
-        context_hosts = capture_with_info(*KAMAL.builder.context_hosts).split("\n")
+        # Get the command here to ensure the Dir.chdir doesn't interfere with it
+        push = KAMAL.builder.push(cli.options[:output])
 
-        if context_hosts != KAMAL.builder.config_context_hosts
-          warn "Context hosts have changed, so re-creating builder, was: #{context_hosts.join(", ")}], now: #{KAMAL.builder.config_context_hosts.join(", ")}"
-          cli.remove
-          cli.create
-        end
-      rescue SSHKit::Command::Failed => e
-        if e.message =~ /(context not found|no builder|does not exist)/
-          warn "Missing compatible builder, so creating a new one first"
-          cli.create
-        else
-          raise
+        KAMAL.with_verbosity(:debug) do
+          Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
         end
       end
-
-      KAMAL.with_verbosity(:debug) do
-        Dir.chdir(KAMAL.config.builder.build_directory) { execute *push }
-      end
     end
   end
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
+    login_to_registry_remotely
+
     if (first_hosts = mirror_hosts).any?
       #  Pull on a single host per mirror first to seed them
       say "Pulling image on #{first_hosts.join(", ")} to seed the #{"mirror".pluralize(first_hosts.count)}...", :magenta
       pull_on_hosts(first_hosts)
       say "Pulling image on remaining hosts...", :magenta
-      pull_on_hosts(KAMAL.hosts - first_hosts)
+      pull_on_hosts(KAMAL.app_hosts - first_hosts)
     else
-      pull_on_hosts(KAMAL.hosts)
+      pull_on_hosts(KAMAL.app_hosts)
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    if (remote_host = KAMAL.config.builder.remote_host)
+    if (remote_host = KAMAL.config.builder.remote)
       connect_to_remote_host(remote_host)
     end
 
@@ -107,21 +117,42 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
   end
 
-  private
-    def verify_local_dependencies
-      run_locally do
-        begin
-          execute *KAMAL.builder.ensure_local_dependencies_installed
-        rescue SSHKit::Command::Failed => e
-          build_error = e.message =~ /command not found/ ?
-            "Docker is not installed locally" :
-            "Docker buildx plugin is not installed locally"
+  desc "dev", "Build using the working directory, tag it as dirty, and push to local image store."
+  option :output, type: :string, default: "docker", banner: "export_type", desc: "Exported type for the build result, and may be any exported type supported by 'buildx --output'."
+  def dev
+    cli = self
+
+    ensure_docker_installed
 
-          raise BuildError, build_error
+    docker_included_files = Set.new(Kamal::Docker.included_files)
+    git_uncommitted_files = Set.new(Kamal::Git.uncommitted_files)
+    git_untracked_files = Set.new(Kamal::Git.untracked_files)
+
+    docker_uncommitted_files = docker_included_files & git_uncommitted_files
+    if docker_uncommitted_files.any?
+      say "WARNING: Files with uncommitted changes will be present in the dev container:", :yellow
+      docker_uncommitted_files.sort.each { |f| say "  #{f}", :yellow }
+      say
+    end
+
+    docker_untracked_files = docker_included_files & git_untracked_files
+    if docker_untracked_files.any?
+      say "WARNING: Untracked files will be present in the dev container:", :yellow
+      docker_untracked_files.sort.each { |f| say "  #{f}", :yellow }
+      say
+    end
+
+    with_env(KAMAL.config.builder.secrets) do
+      run_locally do
+        build = KAMAL.builder.push(cli.options[:output], tag_as_dirty: true)
+        KAMAL.with_verbosity(:debug) do
+          execute(*build)
         end
       end
     end
+  end
 
+  private
     def connect_to_remote_host(remote_host)
       remote_uri = URI.parse(remote_host)
       if remote_uri.scheme == "ssh"
@@ -136,9 +167,9 @@ class Kamal::Cli::Build < Kamal::Cli::Base
     end
 
     def mirror_hosts
-      if KAMAL.hosts.many?
+      if KAMAL.app_hosts.many?
         mirror_hosts = Concurrent::Hash.new
-        on(KAMAL.hosts) do |host|
+        on(KAMAL.app_hosts) do |host|
           first_mirror = capture_with_info(*KAMAL.builder.first_mirror).strip.presence
           mirror_hosts[first_mirror] ||= host.to_s if first_mirror
         rescue SSHKit::Command::Failed => e
@@ -158,4 +189,16 @@ class Kamal::Cli::Build < Kamal::Cli::Base
         execute *KAMAL.builder.validate_image
       end
     end
+
+    def login_to_registry_locally
+      run_locally do
+        execute *KAMAL.registry.login
+      end
+    end
+
+    def login_to_registry_remotely
+      on(KAMAL.app_hosts) do
+        execute *KAMAL.registry.login
+      end
+    end
 end

data/lib/kamal/cli/healthcheck/barrier.rb

@@ -1,3 +1,5 @@
+require "concurrent/ivar"
+
 class Kamal::Cli::Healthcheck::Barrier
   def initialize
     @ivar = Concurrent::IVar.new

data/lib/kamal/cli/healthcheck/poller.rb

@@ -1,51 +1,30 @@
 module Kamal::Cli::Healthcheck::Poller
   extend self
 
-  TRAEFIK_UPDATE_DELAY = 5
-
-
-  def wait_for_healthy(pause_after_ready: false, &block)
+  def wait_for_healthy(role, &block)
     attempt = 1
-    max_attempts = KAMAL.config.healthcheck.max_attempts
+    timeout_at = Time.now + KAMAL.config.deploy_timeout
+    readiness_delay = KAMAL.config.readiness_delay
 
     begin
-      case status = block.call
-      when "healthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      when "running" # No health check configured
-        sleep KAMAL.config.readiness_delay if pause_after_ready
-      else
-        raise Kamal::Cli::Healthcheck::Error, "container not ready (#{status})"
-      end
-    rescue Kamal::Cli::Healthcheck::Error => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
-        attempt += 1
-        retry
-      else
-        raise
+      status = block.call
+
+      if status == "running"
+        # Wait for the readiness delay and confirm it is still running
+        if readiness_delay > 0
+          info "Container is running, waiting for readiness delay of #{readiness_delay} seconds"
+          sleep readiness_delay
+          status = block.call
+        end
       end
-    end
 
-    info "Container is healthy!"
-  end
-
-  def wait_for_unhealthy(pause_after_ready: false, &block)
-    attempt = 1
-    max_attempts = KAMAL.config.healthcheck.max_attempts
-
-    begin
-      case status = block.call
-      when "unhealthy"
-        sleep TRAEFIK_UPDATE_DELAY if pause_after_ready
-      else
-        raise Kamal::Cli::Healthcheck::Error, "container not unhealthy (#{status})"
+      unless %w[ running healthy ].include?(status)
+        raise Kamal::Cli::Healthcheck::Error, "container not ready after #{KAMAL.config.deploy_timeout} seconds (#{status})"
       end
     rescue Kamal::Cli::Healthcheck::Error => e
-      if attempt <= max_attempts
-        info "#{e.message}, retrying in #{attempt}s (attempt #{attempt}/#{max_attempts})..."
-        sleep attempt
+      time_left = timeout_at - Time.now
+      if time_left > 0
+        sleep [ attempt, time_left ].min
         attempt += 1
         retry
       else
@@ -53,7 +32,7 @@ module Kamal::Cli::Healthcheck::Poller
       end
     end
 
-    info "Container is unhealthy!"
+    info "Container is healthy!"
   end
 
   private

data/lib/kamal/cli/lock.rb

@@ -3,7 +3,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def status
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         puts capture_with_debug(*KAMAL.lock.status)
       end
     end
@@ -13,9 +12,10 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   option :message, aliases: "-m", type: :string, desc: "A lock message", required: true
   def acquire
     message = options[:message]
+    ensure_run_directory
+
     raise_if_locked do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug
       end
       say "Acquired the deploy lock"
@@ -26,7 +26,6 @@ class Kamal::Cli::Lock < Kamal::Cli::Base
   def release
     handle_missing_lock do
       on(KAMAL.primary_host) do
-        execute *KAMAL.server.ensure_run_directory
         execute *KAMAL.lock.release, verbosity: :debug
       end
       say "Released the deploy lock"

data/lib/kamal/cli/main.rb

@@ -9,25 +9,17 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         say "Ensure Docker is installed...", :magenta
         invoke "kamal:cli:server:bootstrap", [], invoke_options
 
-        say "Evaluate and push env files...", :magenta
-        invoke "kamal:cli:main:envify", [], invoke_options
-        invoke "kamal:cli:env:push", [], invoke_options
-
-        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options
-        deploy
+        deploy(boot_accessories: true)
       end
     end
   end
 
   desc "deploy", "Deploy app to servers"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
-  def deploy
+  def deploy(boot_accessories: false)
     runtime = print_runtime do
       invoke_options = deploy_options
 
-      say "Log into image registry...", :magenta
-      invoke "kamal:cli:registry:login", [], invoke_options.merge(skip_local: options[:skip_push])
-
       if options[:skip_push]
         say "Pull app image...", :magenta
         invoke "kamal:cli:build:pull", [], invoke_options
@@ -37,10 +29,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
 
       with_lock do
-        run_hook "pre-deploy"
+        run_hook "pre-deploy", secrets: true
 
-        say "Ensure Traefik is running...", :magenta
-        invoke "kamal:cli:traefik:boot", [], invoke_options
+        say "Ensure kamal-proxy is running...", :magenta
+        invoke "kamal:cli:proxy:boot", [], invoke_options
+
+        invoke "kamal:cli:accessory:boot", [ "all" ], invoke_options if boot_accessories
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -52,10 +46,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
-  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting Traefik, pruning, and registry login"
+  desc "redeploy", "Deploy app to servers without bootstrapping servers, starting kamal-proxy and pruning"
   option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip image build and push"
   def redeploy
     runtime = print_runtime do
@@ -70,7 +64,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
 
       with_lock do
-        run_hook "pre-deploy"
+        run_hook "pre-deploy", secrets: true
 
         say "Detect stale containers...", :magenta
         invoke "kamal:cli:app:stale_containers", [], invoke_options.merge(stop: true)
@@ -79,7 +73,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s
   end
 
   desc "rollback [VERSION]", "Rollback app to VERSION"
@@ -93,7 +87,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
         old_version = nil
 
         if container_available?(version)
-          run_hook "pre-deploy"
+          run_hook "pre-deploy", secrets: true
 
           invoke "kamal:cli:app:boot", [], invoke_options.merge(version: version)
           rolled_back = true
@@ -103,12 +97,12 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       end
     end
 
-    run_hook "post-deploy", runtime: runtime.round if rolled_back
+    run_hook "post-deploy", secrets: true, runtime: runtime.round.to_s if rolled_back
   end
 
   desc "details", "Show details about all containers"
   def details
-    invoke "kamal:cli:traefik:details"
+    invoke "kamal:cli:proxy:details"
     invoke "kamal:cli:app:details"
     invoke "kamal:cli:accessory:details", [ "all" ]
   end
@@ -127,7 +121,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
-  desc "docs", "Show Kamal documentation for configuration setting"
+  desc "docs [SECTION]", "Show Kamal configuration documentation"
   def docs(section = nil)
     case section
     when NilClass
@@ -139,7 +133,7 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     puts "No documentation found for #{section}"
   end
 
-  desc "init", "Create config stub in config/deploy.yml and env stub in .env"
+  desc "init", "Create config stub in config/deploy.yml and secrets stub in .kamal"
   option :bundle, type: :boolean, default: false, desc: "Add Kamal to the Gemfile and create a bin/kamal binstub"
   def init
     require "fileutils"
@@ -152,9 +146,10 @@ class Kamal::Cli::Main < Kamal::Cli::Base
       puts "Created configuration file in config/deploy.yml"
     end
 
-    unless (deploy_file = Pathname.new(File.expand_path(".env"))).exist?
-      FileUtils.cp_r Pathname.new(File.expand_path("templates/template.env", __dir__)), deploy_file
-      puts "Created .env file"
+    unless (secrets_file = Pathname.new(File.expand_path(".kamal/secrets"))).exist?
+      FileUtils.mkdir_p secrets_file.dirname
+      FileUtils.cp_r Pathname.new(File.expand_path("templates/secrets", __dir__)), secrets_file
+      puts "Created .kamal/secrets file"
     end
 
     unless (hooks_dir = Pathname.new(File.expand_path(".kamal/hooks"))).exist?
@@ -179,44 +174,50 @@ class Kamal::Cli::Main < Kamal::Cli::Base
     end
   end
 
-  desc "envify", "Create .env by evaluating .env.erb (or .env.staging.erb -> .env.staging when using -d staging)"
-  option :skip_push, aliases: "-P", type: :boolean, default: false, desc: "Skip .env file push"
-  def envify
-    if destination = options[:destination]
-      env_template_path = ".env.#{destination}.erb"
-      env_path          = ".env.#{destination}"
-    else
-      env_template_path = ".env.erb"
-      env_path          = ".env"
-    end
-
-    if Pathname.new(File.expand_path(env_template_path)).exist?
-      # Ensure existing env doesn't pollute template evaluation
-      content = with_original_env { ERB.new(File.read(env_template_path), trim_mode: "-").result }
-      File.write(env_path, content, perm: 0600)
-
-      unless options[:skip_push]
-        reload_env
-        invoke "kamal:cli:env:push", options
-      end
-    else
-      puts "Skipping envify (no #{env_template_path} exist)"
-    end
-  end
-
-  desc "remove", "Remove Traefik, app, accessories, and registry session from servers"
+  desc "remove", "Remove kamal-proxy, app, accessories, and registry session from servers"
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def remove
     confirming "This will remove all containers and images. Are you sure?" do
       with_lock do
-        invoke "kamal:cli:traefik:remove", [], options.without(:confirmed)
         invoke "kamal:cli:app:remove", [], options.without(:confirmed)
+        invoke "kamal:cli:proxy:remove", [], options.without(:confirmed)
         invoke "kamal:cli:accessory:remove", [ "all" ], options
         invoke "kamal:cli:registry:logout", [], options.without(:confirmed).merge(skip_local: true)
       end
     end
   end
 
+  desc "upgrade", "Upgrade from Kamal 1.x to 2.0"
+  option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
+  option :rolling, type: :boolean, default: false, desc: "Upgrade one host at a time"
+  def upgrade
+    confirming "This will replace Traefik with kamal-proxy and restart all accessories" do
+      with_lock do
+        if options[:rolling]
+          KAMAL.hosts.each do |host|
+            KAMAL.with_specific_hosts(host) do
+              say "Upgrading #{host}...", :magenta
+              if KAMAL.app_hosts.include?(host)
+                invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Proxy)
+              end
+              if KAMAL.accessory_hosts.include?(host)
+                invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true, rolling: false)
+                reset_invocation(Kamal::Cli::Accessory)
+              end
+              say "Upgraded #{host}", :magenta
+            end
+          end
+        else
+          say "Upgrading all hosts...", :magenta
+          invoke "kamal:cli:proxy:upgrade", [], options.merge(confirmed: true)
+          invoke "kamal:cli:accessory:upgrade", [ "all" ], options.merge(confirmed: true)
+          say "Upgraded all hosts", :magenta
+        end
+      end
+    end
+  end
+
   desc "version", "Show Kamal version"
   def version
     puts Kamal::VERSION
@@ -231,28 +232,28 @@ class Kamal::Cli::Main < Kamal::Cli::Base
   desc "build", "Build application image"
   subcommand "build", Kamal::Cli::Build
 
-  desc "env", "Manage environment files"
-  subcommand "env", Kamal::Cli::Env
-
   desc "lock", "Manage the deploy lock"
   subcommand "lock", Kamal::Cli::Lock
 
+  desc "proxy", "Manage kamal-proxy"
+  subcommand "proxy", Kamal::Cli::Proxy
+
   desc "prune", "Prune old application images and containers"
   subcommand "prune", Kamal::Cli::Prune
 
   desc "registry", "Login and -out of the image registry"
   subcommand "registry", Kamal::Cli::Registry
 
+  desc "secrets", "Helpers for extracting secrets"
+  subcommand "secrets", Kamal::Cli::Secrets
+
   desc "server", "Bootstrap servers with curl and Docker"
   subcommand "server", Kamal::Cli::Server
 
-  desc "traefik", "Manage Traefik load balancer"
-  subcommand "traefik", Kamal::Cli::Traefik
-
   private
     def container_available?(version)
       begin
-        on(KAMAL.hosts) do
+        on(KAMAL.app_hosts) do
           KAMAL.roles_on(host).each do |role|
             container_id = capture_with_info(*KAMAL.app(role: role, host: host).container_id_for_version(version))
             raise "Container not found" unless container_id.present?