dash 3.1.3 → 3.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f4a83d0c755177ff76a6adbe519a705cff0c2fa32ea8a5dc4c71fbd2b10a74b3
-  data.tar.gz: c80e2fedd3a5fb8b7e6949f384698e461b1073172ed9d99c8ccd1d7f0391d4a3
+  metadata.gz: 8c3e7210d642d1f0fc0d9a414cdf74a0c4a1ec9e09cf2f534c1cb1f6b02342a5
+  data.tar.gz: d91f21902a85f56d76bfde689a75d6c3d5b10aa587138de732943e3a0015433d
 SHA512:
-  metadata.gz: e6ea280741d6c78b1b0ed7e6762f536dd4968fb2d9575dd347d07376cfc1128f4bbed91a2c3dc2d09a1620b8a24202385854c1ded2f74d7a6093fa8955ac7b21
-  data.tar.gz: f3e9e2f1946fd99356a349a4d6148c4aa457ae4227ce08171f69f9811c27b8572bf8f4524f63631c533c4b01ba26ef5e4033be738fc1bd7f282a3e961d7eb4db
+  metadata.gz: e62792a82328365e87efd4428cfab12808281fc617157f8699e757962b742988560ff66ebdd0c5fa7f36f6071b1a46441284038c442e74724ec0b1caa354f5c9
+  data.tar.gz: f328a6e03c4d302170ad406a6550ddd95987622b26af74442945b25cec44081074b995a633b97f546bb873b97e3b9b3905c78973526698a5e0dd4d5635f6dd13

data/lib/kamal/cli/base.rb

@@ -125,9 +125,13 @@ module Kamal::Cli
         puts "  Finished all in #{sprintf("%.1f seconds", runtime)}"
       end
 
-      def modify(lock: false)
+      # Deploy lock outside, server lock inside. Every caller acquires in that
+      # order, and deploy locks are unique per destination, so no cycle forms.
+      def modify(lock: false, server_lock: false)
         KAMAL.modify(command: command, subcommand: subcommand) do
-          lock ? with_lock { yield } : yield
+          guarded = -> { server_lock ? with_server_lock { yield } : yield }
+
+          lock ? with_lock(&guarded) : guarded.call
         end
       end
 
@@ -142,6 +146,95 @@ module Kamal::Cli
         raw ? KAMAL.with_verbosity(:error, &block) : block.call
       end
 
+      # kamal-proxy is one container per host, shared by every destination
+      # deployed there, but the deploy lock is per-destination — so two
+      # destinations deploying at once take different locks and both mutate the
+      # same proxy. Anything touching the proxy takes this lock as well.
+      def with_server_lock
+        if KAMAL.holding_server_lock?
+          yield
+        else
+          acquire_server_lock
+
+          begin
+            yield
+          ensure
+            release_server_lock
+          end
+        end
+      end
+
+      # Always waits rather than failing: the guarded work is short, and
+      # aborting the second destination would defeat running them in parallel.
+      #
+      # Hosts are taken one at a time and rolled back on contention. Taking them
+      # in one `on(hosts)` sweep would leave the locks we did win in place, and
+      # every retry would then collide with itself and wait out the timeout.
+      def acquire_server_lock
+        ensure_run_directory
+
+        timeout = KAMAL.lock_wait_timeout
+        interval = KAMAL.lock_wait_interval
+        deadline = Time.now + timeout
+        details_shown = false
+
+        say "Acquiring the server lock...", :magenta
+
+        loop do
+          held = []
+
+          begin
+            server_lock_hosts.each do |host|
+              execute_lock_acquire(AUTOMATIC_DEPLOY_LOCK_MESSAGE, lock: KAMAL.server_lock, hosts: host)
+              held << host
+            end
+
+            break
+          rescue LockHeldError
+            release_server_lock_on(held)
+
+            unless details_shown
+              say "Server lock is held by:", :magenta
+              puts capture_lock_status(lock: KAMAL.server_lock, hosts: server_lock_hosts.first)
+              details_shown = true
+            end
+
+            remaining = (deadline - Time.now).to_i
+            if remaining <= 0
+              say "Timed out after #{timeout}s waiting for the server lock", :red
+              raise LockError, "Timed out waiting for server lock"
+            end
+
+            say "Waiting #{interval}s for the server lock (#{remaining}s remaining)...", :magenta
+            sleep [ interval, remaining ].min
+          end
+        end
+
+        KAMAL.holding_server_lock = true
+      end
+
+      def release_server_lock
+        say "Releasing the server lock...", :magenta
+        release_server_lock_on(server_lock_hosts)
+
+        KAMAL.holding_server_lock = false
+      end
+
+      # Only ever called with hosts this process actually locked, so a missing
+      # directory means someone already cleaned up, not that we may delete
+      # another deploy's lock.
+      def release_server_lock_on(hosts)
+        Array(hosts).each do |host|
+          execute_lock_release(lock: KAMAL.server_lock, hosts: host)
+        rescue LockMissingError
+          nil
+        end
+      end
+
+      def server_lock_hosts
+        KAMAL.proxy_hosts.presence || KAMAL.hosts
+      end
+
       def with_lock
         if KAMAL.holding_lock?
           yield
@@ -239,23 +332,23 @@ module Kamal::Cli
         raise LockError, "Deploy lock found. Run 'kamal lock help' for more information"
       end
 
-      def execute_lock_acquire(message)
-        on(KAMAL.primary_host) { execute *KAMAL.lock.acquire(message, KAMAL.config.version), verbosity: :debug }
+      def execute_lock_acquire(message, lock: KAMAL.lock, hosts: KAMAL.primary_host)
+        on(hosts) { execute *lock.acquire(message, KAMAL.config.version), verbosity: :debug }
       rescue SSHKit::Runner::ExecuteError => e
         raise LockHeldError if e.message =~ /cannot create directory/
         raise
       end
 
-      def execute_lock_release
-        on(KAMAL.primary_host) { execute *KAMAL.lock.release, verbosity: :debug }
+      def execute_lock_release(lock: KAMAL.lock, hosts: KAMAL.primary_host)
+        on(hosts) { execute *lock.release, verbosity: :debug }
       rescue SSHKit::Runner::ExecuteError => e
         raise LockMissingError if e.message =~ /No such file or directory/
         raise
       end
 
-      def capture_lock_status
+      def capture_lock_status(lock: KAMAL.lock, hosts: KAMAL.primary_host)
         status = nil
-        on(KAMAL.primary_host) { status = capture_with_debug(*KAMAL.lock.status) }
+        on(hosts) { status = capture_with_debug(*lock.status) }
         status
       rescue SSHKit::Runner::ExecuteError => e
         raise LockMissingError if e.message =~ /No such file or directory/

data/lib/kamal/cli/proxy.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Proxy < Kamal::Cli::Base
   desc "boot", "Boot proxy on servers"
   def boot
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.hosts) do |host|
         execute *KAMAL.docker.create_network
       rescue SSHKit::Command::Failed => e
@@ -215,7 +215,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
   option :confirmed, aliases: "-y", type: :boolean, default: false, desc: "Proceed without confirmation question"
   def reboot
     confirming "This will cause a brief outage on each host. Are you sure?" do
-      modify(lock: true) do
+      modify(lock: true, server_lock: true) do
         # Skip proxy on loadbalancer host - it will be handled by loadbalancer reboot
         proxy_hosts = KAMAL.proxy_hosts
         if KAMAL.config.proxy.loadbalancer_on_proxy_host?
@@ -292,7 +292,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "start", "Start existing proxy container on servers"
   def start
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.proxy_hosts) do |host|
         execute *KAMAL.auditor.record("Started proxy"), verbosity: :debug
         execute *KAMAL.proxy(host).start
@@ -309,7 +309,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "stop", "Stop existing proxy container on servers"
   def stop
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.proxy_hosts) do |host|
         execute *KAMAL.auditor.record("Stopped proxy"), verbosity: :debug
         execute *KAMAL.proxy(host).stop, raise_on_non_zero_exit: false
@@ -329,7 +329,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "restart", "Restart existing proxy container on servers"
   def restart
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       stop
       start
     end
@@ -377,7 +377,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
   desc "remove", "Remove proxy container and image from servers"
   option :force, type: :boolean, default: false, desc: "Force removing proxy when apps are still installed"
   def remove
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       if removal_allowed?(options[:force])
         stop
         remove_container
@@ -510,7 +510,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
     # Under the deploy lock: a concurrent deploy could boot or reboot the
     # proxy mid-export, and the offline read below would archive a torn store.
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(cert_store_host) do |host|
         execute *KAMAL.auditor.record("Exported the proxy certificate store"), verbosity: :debug
 
@@ -550,7 +550,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
     force, verify = options[:force], options[:verify]
     load_balancing = KAMAL.config.proxy.load_balancing?
 
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(cert_store_host) do |host|
         commands = load_balancing ? KAMAL.loadbalancer : KAMAL.proxy(host)
 
@@ -583,7 +583,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "remove_container", "Remove proxy container from servers", hide: true
   def remove_container
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.proxy_hosts) do
         execute *KAMAL.auditor.record("Removed proxy container"), verbosity: :debug
         execute *KAMAL.proxy(host).remove_container
@@ -600,7 +600,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "remove_image", "Remove proxy image from servers", hide: true
   def remove_image
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.proxy_hosts) do
         execute *KAMAL.auditor.record("Removed proxy image"), verbosity: :debug
         execute *KAMAL.proxy(host).remove_image
@@ -617,7 +617,7 @@ class Kamal::Cli::Proxy < Kamal::Cli::Base
 
   desc "remove_proxy_directory", "Remove the proxy directory from servers", hide: true
   def remove_proxy_directory
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.proxy_hosts) do
         execute *KAMAL.proxy(host).remove_proxy_directory, raise_on_non_zero_exit: false
       end

data/lib/kamal/cli/prune.rb

@@ -1,7 +1,7 @@
 class Kamal::Cli::Prune < Kamal::Cli::Base
   desc "all", "Prune unused images and stopped containers"
   def all
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       containers
       images
     end
@@ -9,7 +9,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
 
   desc "images", "Prune unused images"
   def images
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.hosts) do
         execute *KAMAL.auditor.record("Pruned images"), verbosity: :debug
         execute *KAMAL.prune.dangling_images
@@ -24,7 +24,7 @@ class Kamal::Cli::Prune < Kamal::Cli::Base
     retain = options.fetch(:retain, KAMAL.config.retain_containers)
     raise "retain must be at least 1" if retain < 1
 
-    modify(lock: true) do
+    modify(lock: true, server_lock: true) do
       on(KAMAL.hosts) do |host|
         execute *KAMAL.auditor.record("Pruned containers"), verbosity: :debug
 

data/lib/kamal/commander.rb

@@ -5,7 +5,7 @@ require "active_support/broadcast_logger"
 require "active_support/notifications"
 
 class Kamal::Commander
-  attr_accessor :verbosity, :holding_lock, :connected, :logging, :lock_wait, :lock_wait_timeout, :lock_wait_interval
+  attr_accessor :verbosity, :holding_lock, :holding_server_lock, :connected, :logging, :lock_wait, :lock_wait_timeout, :lock_wait_interval
   attr_reader :specific_roles, :specific_hosts
   delegate :hosts, :roles, :primary_host, :primary_role, :roles_on, :app_hosts, :proxy_hosts, :accessory_hosts, to: :specifics
 
@@ -16,6 +16,7 @@ class Kamal::Commander
   def reset
     self.verbosity = :info
     self.holding_lock = ENV["KAMAL_LOCK"] == "true"
+    self.holding_server_lock = ENV["KAMAL_SERVER_LOCK"] == "true"
     self.connected = false
     self.logging = false
     self.lock_wait = false
@@ -109,6 +110,12 @@ class Kamal::Commander
     @commands[:lock] ||= Kamal::Commands::Lock.new(config)
   end
 
+  # Guards the kamal-proxy container, which a host runs once for every
+  # destination deployed onto it. Every destination contends for this one.
+  def server_lock
+    @commands[:server_lock] ||= Kamal::Commands::Lock.new(config, scope: :server)
+  end
+
   def proxy(host)
     Kamal::Commands::Proxy.new(config, host: host)
   end
@@ -174,6 +181,10 @@ class Kamal::Commander
     output_logger << "#{line}\n" if logging
   end
 
+  def holding_server_lock?
+    self.holding_server_lock
+  end
+
   def holding_lock?
     self.holding_lock
   end

data/lib/kamal/commands/lock.rb

@@ -3,6 +3,21 @@ require "time"
 require "base64"
 
 class Kamal::Commands::Lock < Kamal::Commands::Base
+  # The deploy lock is scoped to service+destination, so two destinations can
+  # deploy at once. Server-scoped resources — above all the single kamal-proxy
+  # container a host runs for every destination on it — need a lock that every
+  # destination contends for, hence :server.
+  SCOPES = %i[ destination server ].freeze
+
+  attr_reader :scope
+
+  def initialize(config, scope: :destination)
+    raise ArgumentError, "Unknown lock scope: #{scope}" unless SCOPES.include?(scope)
+
+    super(config)
+    @scope = scope
+  end
+
   def acquire(message, version)
     combine \
       [ :mkdir, lock_dir ],
@@ -45,9 +60,16 @@ class Kamal::Commands::Lock < Kamal::Commands::Base
     end
 
     def lock_dir
-      dir_name = [ "lock", config.service, config.destination ].compact.join("-")
+      File.join(config.run_directory, lock_dir_name)
+    end
 
-      File.join(config.run_directory, dir_name)
+    # A server-scoped lock deliberately carries neither service nor destination:
+    # every deploy reaching this host must collide on the same directory.
+    def lock_dir_name
+      case scope
+      when :server      then "lock-server"
+      when :destination then [ "lock", config.service, config.destination ].compact.join("-")
+      end
     end
 
     def lock_details_file

data/lib/kamal/version.rb

@@ -1,3 +1,3 @@
 module Kamal
-  VERSION = "3.1.3"
+  VERSION = "3.1.4"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dash
 version: !ruby/object:Gem::Version
-  version: 3.1.3
+  version: 3.1.4
 platform: ruby
 authors:
 - Mikael Henriksson