mrsk 0.9.0 → 0.10.0

data/lib/mrsk/cli/app.rb

@@ -1,32 +1,31 @@
 class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    say "Get most recent version available as an image...", :magenta unless options[:version]
-    using_version(options[:version] || most_recent_version_available) do |version|
-      say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
+    with_lock do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
-      cli = self
-      
-      MRSK.config.roles.each do |role|
-        on(role.hosts) do |host|
-          execute *MRSK.auditor.record("Booted app version #{version}"), verbosity: :debug
+        cli = self
 
-          begin
-            old_version = capture_with_info(*MRSK.app.current_running_version).strip
-            execute *MRSK.app.run(role: role.name)
-            sleep MRSK.config.readiness_delay
-            execute *MRSK.app.stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
-
-          rescue SSHKit::Command::Failed => e
-            if e.message =~ /already in use/
-              error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
-              execute *MRSK.auditor.record("Rebooted app version #{version}"), verbosity: :debug
-
-              execute *MRSK.app.stop(version: version)
-              execute *MRSK.app.remove_container(version: version)
-              execute *MRSK.app.run(role: role.name)
-            else
-              raise
+        on(MRSK.hosts) do |host|
+          roles = MRSK.roles_on(host)
+
+          roles.each do |role|
+            execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
+
+            begin
+              if capture_with_info(*MRSK.app(role: role).container_id_for_version(version)).present?
+                tmp_version = "#{version}_#{SecureRandom.hex(8)}"
+                info "Renaming container #{version} to #{tmp_version} as already deployed on #{host}"
+                execute *MRSK.auditor(role: role).record("Renaming container #{version} to #{tmp_version}"), verbosity: :debug
+                execute *MRSK.app(role: role).rename_container(version: version, new_version: tmp_version)
+              end
+
+              old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
+              execute *MRSK.app(role: role).run
+              sleep MRSK.config.readiness_delay
+              execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
             end
           end
         end
@@ -36,24 +35,42 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
   desc "start", "Start existing app container on servers"
   def start
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Started app version #{MRSK.version}"), verbosity: :debug
-      execute *MRSK.app.start, raise_on_non_zero_exit: false
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
+          execute *MRSK.app(role: role).start, raise_on_non_zero_exit: false
+        end
+      end
     end
   end
 
   desc "stop", "Stop app container on servers"
   def stop
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Stopped app"), verbosity: :debug
-      execute *MRSK.app.stop, raise_on_non_zero_exit: false
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
+          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
+        end
+      end
     end
   end
 
   # FIXME: Drop in favor of just containers?
   desc "details", "Show details about app containers"
   def details
-    on(MRSK.hosts) { |host| puts_by_host host, capture_with_info(*MRSK.app.info) }
+    on(MRSK.hosts) do |host|
+      roles = MRSK.roles_on(host)
+
+      roles.each do |role|
+        puts_by_host host, capture_with_info(*MRSK.app(role: role).info)
+      end
+    end
   end
 
   desc "exec [CMD]", "Execute a custom command on servers (use --help to show options)"
@@ -65,12 +82,12 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       say "Get current version of running container...", :magenta unless options[:version]
       using_version(options[:version] || current_running_version) do |version|
         say "Launching interactive command with version #{version} via SSH from existing container on #{MRSK.primary_host}...", :magenta
-        run_locally { exec MRSK.app.execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) }
+        run_locally { exec MRSK.app(role: "web").execute_in_existing_container_over_ssh(cmd, host: MRSK.primary_host) }
       end
 
     when options[:interactive]
       say "Get most recent version available as an image...", :magenta unless options[:version]
-      using_version(options[:version] || most_recent_version_available) do |version|
+      using_version(version_or_latest) do |version|
         say "Launching interactive command with version #{version} via SSH from new container on #{MRSK.primary_host}...", :magenta
         run_locally { exec MRSK.app.execute_in_new_container_over_ssh(cmd, host: MRSK.primary_host) }
       end
@@ -81,14 +98,18 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
         say "Launching command with version #{version} from existing container...", :magenta
 
         on(MRSK.hosts) do |host|
-          execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
-          puts_by_host host, capture_with_info(*MRSK.app.execute_in_existing_container(cmd))
+          roles = MRSK.roles_on(host)
+
+          roles.each do |role|
+            execute *MRSK.auditor(role: role).record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
+            puts_by_host host, capture_with_info(*MRSK.app(role: role).execute_in_existing_container(cmd))
+          end
         end
       end
 
     else
       say "Get most recent version available as an image...", :magenta unless options[:version]
-      using_version(options[:version] || most_recent_version_available) do |version|
+      using_version(version_or_latest) do |version|
         say "Launching command with version #{version} from new container...", :magenta
         on(MRSK.hosts) do |host|
           execute *MRSK.auditor.record("Executed cmd '#{cmd}' on app version #{version}"), verbosity: :debug
@@ -121,18 +142,26 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
     if options[:follow]
       run_locally do
         info "Following logs on #{MRSK.primary_host}..."
-        info MRSK.app.follow_logs(host: MRSK.primary_host, grep: grep)
-        exec MRSK.app.follow_logs(host: MRSK.primary_host, grep: grep)
+
+        MRSK.specific_roles ||= ["web"]
+        role = MRSK.roles_on(MRSK.primary_host).first
+
+        info MRSK.app(role: role).follow_logs(host: MRSK.primary_host, grep: grep)
+        exec MRSK.app(role: role).follow_logs(host: MRSK.primary_host, grep: grep)
       end
     else
       since = options[:since]
       lines = options[:lines].presence || ((since || grep) ? nil : 100) # Default to 100 lines if since or grep isn't set
 
       on(MRSK.hosts) do |host|
-        begin
-          puts_by_host host, capture_with_info(*MRSK.app.logs(since: since, lines: lines, grep: grep))
-        rescue SSHKit::Command::Failed
-          puts_by_host host, "Nothing found"
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          begin
+            puts_by_host host, capture_with_info(*MRSK.app(role: role).logs(since: since, lines: lines, grep: grep))
+          rescue SSHKit::Command::Failed
+            puts_by_host host, "Nothing found"
+          end
         end
       end
     end
@@ -140,32 +169,48 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    stop
-    remove_containers
-    remove_images
+    with_lock do
+      stop
+      remove_containers
+      remove_images
+    end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Removed app container with version #{version}"), verbosity: :debug
-      execute *MRSK.app.remove_container(version: version)
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Removed app container with version #{version}"), verbosity: :debug
+          execute *MRSK.app(role: role).remove_container(version: version)
+        end
+      end
     end
   end
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Removed all app containers"), verbosity: :debug
-      execute *MRSK.app.remove_containers
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
+
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Removed all app containers"), verbosity: :debug
+          execute *MRSK.app(role: role).remove_containers
+        end
+      end
     end
   end
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
-      execute *MRSK.app.remove_images
+    with_lock do
+      on(MRSK.hosts) do
+        execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
+        execute *MRSK.app.remove_images
+      end
     end
   end
 
@@ -189,20 +234,13 @@ class Mrsk::Cli::App < Mrsk::Cli::Base
       end
     end
 
-    def most_recent_version_available(host: MRSK.primary_host)
-      version = nil
-      on(host) { version = capture_with_info(*MRSK.app.most_recent_version_from_available_images).strip }
-      
-      if version == "<none>"
-        raise "Most recent image available was not tagged with a version (returned <none>)"
-      else
-        version.presence
-      end
-    end
-
     def current_running_version(host: MRSK.primary_host)
       version = nil
       on(host) { version = capture_with_info(*MRSK.app.current_running_version).strip }
       version.presence
     end
+
+    def version_or_latest
+      options[:version] || "latest"
+    end
 end

data/lib/mrsk/cli/base.rb

@@ -6,6 +6,8 @@ module Mrsk::Cli
   class Base < Thor
     include SSHKit::DSL
 
+    class LockError < StandardError; end
+
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
@@ -25,7 +27,7 @@ module Mrsk::Cli
     def initialize(*)
       super
       load_envs
-      initialize_commander(options)
+      initialize_commander(options_with_subcommand_class_options)
     end
 
     private
@@ -37,16 +39,12 @@ module Mrsk::Cli
         end
       end
 
+      def options_with_subcommand_class_options
+        options.merge(@_initializer.last[:class_options] || {})
+      end
+
       def initialize_commander(options)
         MRSK.tap do |commander|
-          commander.config_file = Pathname.new(File.expand_path(options[:config_file]))
-          commander.destination = options[:destination]
-          commander.version     = options[:version]
-
-          commander.specific_hosts    = options[:hosts]&.split(",")
-          commander.specific_roles    = options[:roles]&.split(",")
-          commander.specific_primary! if options[:primary]
-
           if options[:verbose]
             ENV["VERBOSE"] = "1" # For backtraces via cli/start
             commander.verbosity = :debug
@@ -55,6 +53,15 @@ module Mrsk::Cli
           if options[:quiet]
             commander.verbosity = :error
           end
+
+          commander.configure \
+            config_file: Pathname.new(File.expand_path(options[:config_file])),
+            destination: options[:destination],
+            version: options[:version]
+
+          commander.specific_hosts    = options[:hosts]&.split(",")
+          commander.specific_roles    = options[:roles]&.split(",")
+          commander.specific_primary! if options[:primary]
         end
       end
 
@@ -70,5 +77,35 @@ module Mrsk::Cli
       def audit_broadcast(line)
         run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
       end
+
+      def with_lock
+        acquire_lock
+
+        yield
+      ensure
+        release_lock
+      end
+
+      def acquire_lock
+        if MRSK.lock_count == 0
+          say "Acquiring the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
+        end
+        MRSK.lock_count += 1
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /cannot create directory/
+          invoke "mrsk:cli:lock:status", []
+        end
+
+        raise LockError, "Deploy lock found"
+      end
+
+      def release_lock
+        MRSK.lock_count -= 1
+        if MRSK.lock_count == 0
+          say "Releasing the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.release }
+        end
+      end
   end
 end

data/lib/mrsk/cli/build.rb

@@ -1,26 +1,30 @@
 class Mrsk::Cli::Build < Mrsk::Cli::Base
   desc "deliver", "Build app and push app image to registry then pull image on servers"
   def deliver
-    push
-    pull
+    with_lock do
+      push
+      pull
+    end
   end
 
   desc "push", "Build and push app image to registry"
   def push
-    cli = self
+    with_lock do
+      cli = self
 
-    run_locally do
-      begin
-        MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
-      rescue SSHKit::Command::Failed => e
-        if e.message =~ /(no builder)|(no such file or directory)/
-          error "Missing compatible builder, so creating a new one first"
+      run_locally do
+        begin
+          MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+        rescue SSHKit::Command::Failed => e
+          if e.message =~ /(no builder)|(no such file or directory)/
+            error "Missing compatible builder, so creating a new one first"
 
-          if cli.create
-            MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+            if cli.create
+              MRSK.with_verbosity(:debug) { execute *MRSK.builder.push }
+            end
+          else
+            raise
           end
-        else
-          raise
         end
       end
     end
@@ -28,25 +32,29 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "pull", "Pull app image from registry onto servers"
   def pull
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Pulled image with version #{MRSK.version}"), verbosity: :debug
-      execute *MRSK.builder.clean, raise_on_non_zero_exit: false
-      execute *MRSK.builder.pull
+    with_lock do
+      on(MRSK.hosts) do
+        execute *MRSK.auditor.record("Pulled image with version #{MRSK.config.version}"), verbosity: :debug
+        execute *MRSK.builder.clean, raise_on_non_zero_exit: false
+        execute *MRSK.builder.pull
+      end
     end
   end
 
   desc "create", "Create a build setup"
   def create
-    run_locally do
-      begin
-        debug "Using builder: #{MRSK.builder.name}"
-        execute *MRSK.builder.create
-      rescue SSHKit::Command::Failed => e
-        if e.message =~ /stderr=(.*)/
-          error "Couldn't create remote builder: #{$1}"
-          false
-        else
-          raise
+    with_lock do
+      run_locally do
+        begin
+          debug "Using builder: #{MRSK.builder.name}"
+          execute *MRSK.builder.create
+        rescue SSHKit::Command::Failed => e
+          if e.message =~ /stderr=(.*)/
+            error "Couldn't create remote builder: #{$1}"
+            false
+          else
+            raise
+          end
         end
       end
     end
@@ -54,9 +62,11 @@ class Mrsk::Cli::Build < Mrsk::Cli::Base
 
   desc "remove", "Remove build setup"
   def remove
-    run_locally do
-      debug "Using builder: #{MRSK.builder.name}"
-      execute *MRSK.builder.remove
+    with_lock do
+      run_locally do
+        debug "Using builder: #{MRSK.builder.name}"
+        execute *MRSK.builder.remove
+      end
     end
   end
 

data/lib/mrsk/cli/lock.rb

@@ -0,0 +1,37 @@
+class Mrsk::Cli::Lock < Mrsk::Cli::Base
+  desc "status", "Report lock status"
+  def status
+    handle_missing_lock do
+      on(MRSK.primary_host) { puts capture_with_info(*MRSK.lock.status) }
+    end
+  end
+
+  desc "acquire", "Acquire the deploy lock"
+  option :message, aliases: "-m", type: :string, desc: "A lock mesasge", required: true
+  def acquire
+    message = options[:message]
+    handle_missing_lock do
+      on(MRSK.primary_host) { execute *MRSK.lock.acquire(message, MRSK.config.version) }
+      say "Set the deploy lock"
+    end
+  end
+
+  desc "release", "Release the deploy lock"
+  def release
+    handle_missing_lock do
+      on(MRSK.primary_host) { execute *MRSK.lock.release }
+      say "Removed the deploy lock"
+    end
+  end
+
+  private
+    def handle_missing_lock
+      yield
+    rescue SSHKit::Runner::ExecuteError => e
+      if e.message =~ /No such file or directory/
+        say "There is no deploy lock"
+      else
+        raise
+      end
+    end
+end