moonshot 0.7.6 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d67cb66c5dd32a605e63ad9c2878ba6148786aeb
-  data.tar.gz: ca5974cfe33c7a40d1040984017d5180e67a1da7
+  metadata.gz: a022cf1a54f893264b6ee7487303d79b8c29fb44
+  data.tar.gz: 7425456a02ec0451c431c6dc1863616fae9200e5
 SHA512:
-  metadata.gz: 970d41d6ae16a11f8285381efe2f22a8b9c47fae952e9a860c713bb3aabf57af6e4fc241ccbe72e1640b5a06edea0fc6acdc88db1148a24d03c685eabaecb92f
-  data.tar.gz: dde9fcf2087fdd0aa7918079aa51944b9409567866ab174a0e0c8ab5320b2dbc9ec6adb6257d6edc807b7351063e1c61e640d50c6dee9f0a7d545c0746425370
+  metadata.gz: 8857003d9882c8fa70b559ba182887af90c5bd0b6cf2a62bc60e6bd4109c4e26e1f4ab6fd74bbc5764286cb592a1f70347659f0dfc79b8e13df104cc8356bc40
+  data.tar.gz: 36b9bb90372c8ae2bacf1253061d243c2333f57095fb49f2741de6d2bab602bfaed19652b2bcb86a22df67dc8e45af1ea583b0a760105dc8474c40a570dfa538

data/lib/moonshot/build_mechanism/github_release.rb

@@ -118,7 +118,7 @@ module Moonshot::BuildMechanism
     end
 
     def hub_create_release(semver, commitish, changelog_entry)
-      return if hub_release_exists(semver, commitish)
+      return if hub_release_exists(semver)
 
       message = "#{semver}\n\n#{changelog_entry}"
       cmd = "hub release create #{semver} --commitish=#{commitish}"

data/lib/moonshot/build_mechanism/script.rb

@@ -47,7 +47,7 @@ class Moonshot::BuildMechanism::Script
 
   private
 
-  def run_script(step, env: {}) # rubocop:disable AbcSize
+  def run_script(step, env: {})
     popen2e(env, @script) do |_, out, wait|
       output = []
 
@@ -66,9 +66,7 @@ class Moonshot::BuildMechanism::Script
       end
       unless result.exitstatus == 0
         ilog.error "Build script failed with exit status #{result.exitstatus}!"
-        ilog.error 'Last 10 lines of output follows:'
-        output.pop(10).each { |l| ilog.error l }
-
+        ilog.error output.join("\n")
         step.failure "Build script #{@script} failed with exit status #{result.exitstatus}!"
       end
     end

data/lib/moonshot/build_mechanism/travis_deploy.rb

@@ -1,4 +1,7 @@
 require 'moonshot/shell'
+require 'travis'
+require 'travis/pro'
+require 'travis/client/auto_login'
 
 module Moonshot::BuildMechanism
   # This simply waits for Travis-CI to finish building a job matching the
@@ -12,9 +15,13 @@ module Moonshot::BuildMechanism
 
     attr_reader :output_file
 
-    def initialize(slug, pro: false)
+    def initialize(slug, pro: false, timeout: 900)
       @slug = slug
+      @pro = pro
+      @timeout = timeout
+
       @endpoint = pro ? '--pro' : '--org'
+      @travis_base = @pro ? Travis::Pro : Travis
       @cli_args = "-r #{@slug} #{@endpoint}"
     end
 
@@ -32,6 +39,18 @@ module Moonshot::BuildMechanism
 
     private
 
+    # Authenticates with the proper travis service.
+    def authenticate
+      Travis::Client::AutoLogin.new(@travis_base).authenticate
+    end
+
+    # Retrieves the travis repository.
+    #
+    # @return [Travis::Client::Repository]
+    def repo
+      @repo ||= @travis_base::Repository.find(@slug)
+    end
+
     def find_build_and_job(version)
       job_number = nil
       ilog.start_threaded('Find Travis CI build') do |step|
@@ -78,16 +97,35 @@ module Moonshot::BuildMechanism
       job_number
     end
 
+    # Waits for a job to complete, within the defined timeout.
+    #
+    # @param job_number [String] The job number to wait for.
     def wait_for_job(job_number)
-      cmd = "bundle exec travis logs #{@cli_args} #{job_number}"
-      # This log tailing fails at the end of the file. travis bug.
-      sh_step(cmd, fail: false)
+      authenticate
+
+      # Wait for the job to complete or hit the timeout.
+      start = Time.new
+      job = repo.job(job_number)
+      ilog.start_threaded("Waiting for job #{job_number} to complete.") do |s|
+        while !job.finished? && Time.new - start < @timeout
+          s.continue("Job status: #{job.state}")
+          sleep 10
+          job.reload
+        end
+
+        if job.finished?
+          s.success
+        else
+          s.failure("Job #{job_number} did not complete within time limit of " \
+            "#{@timeout} seconds")
+        end
+      end
     end
 
     def check_build(version)
       cmd = "bundle exec travis show #{@cli_args} #{version}"
       sh_step(cmd) do |step, out|
-        raise "Build didn't pass.\n#{build_out}" \
+        raise "Build didn't pass.\n#{out}" \
           if out =~ /^#(\d+\.\d+) (?!passed).+BUILD=1.+/
 
         step.success("Travis CI build for #{version} passed.")

data/lib/moonshot/cli.rb

@@ -110,8 +110,8 @@ module Moonshot
           config.parameter_strategy = parameter_strategy_factory(parameter_strategy) \
             unless parameter_strategy.nil?
 
-          config.ssh_user = options[:user]
-          config.ssh_identity_file = options[:identity_file]
+          config.ssh_config.ssh_user = options[:user]
+          config.ssh_config.ssh_identity_file = options[:identity_file]
           config.ssh_instance = options[:instance]
           config.ssh_command = options[:command]
           config.ssh_auto_scaling_group_name = options[:auto_scaling_group]

data/lib/moonshot/controller.rb

@@ -1,3 +1,6 @@
+require_relative 'ssh_target_selector'
+require_relative 'ssh_command_builder'
+
 module Moonshot
   # The Controller coordinates and performs all Moonshot actions.
   class Controller # rubocop:disable ClassLength
@@ -82,8 +85,13 @@ module Moonshot
 
     def ssh
       run_plugins(:pre_ssh)
-      stack.ssh
-      run_plugins(:post_ssh)
+      @config.ssh_instance ||= SSHTargetSelector.new(
+        stack, asg_name: @config.ssh_auto_scaling_group_name).choose!
+      cb = SSHCommandBuilder.new(@config.ssh_config, @config.ssh_instance)
+      result = cb.build(@config.ssh_command)
+
+      puts "Opening SSH connection to #{@config.ssh_instance} (#{result.ip})..."
+      exec(result.cmd)
     end
 
     def stack
@@ -94,11 +102,6 @@ module Moonshot
         config.parent_stacks = @config.parent_stacks
         config.show_all_events = @config.show_all_stack_events
         config.parameter_strategy = @config.parameter_strategy
-        config.ssh_user = @config.ssh_user
-        config.ssh_identity_file = @config.ssh_identity_file
-        config.ssh_instance = @config.ssh_instance
-        config.ssh_command = @config.ssh_command
-        config.ssh_auto_scaling_group_name = @config.ssh_auto_scaling_group_name
       end
     end
 

data/lib/moonshot/controller_config.rb

@@ -1,4 +1,5 @@
 require_relative 'default_strategy'
+require_relative 'ssh_config'
 
 module Moonshot
   # Holds configuration for Moonshot::Controller
@@ -15,11 +16,10 @@ module Moonshot
     attr_accessor :plugins
     attr_accessor :show_all_stack_events
     attr_accessor :parameter_strategy
-    attr_accessor :ssh_instance
-    attr_accessor :ssh_identity_file
-    attr_accessor :ssh_user
+    attr_accessor :ssh_config
     attr_accessor :ssh_command
     attr_accessor :ssh_auto_scaling_group_name
+    attr_accessor :ssh_instance
 
     def initialize
       @auto_prefix_stack = true
@@ -29,6 +29,7 @@ module Moonshot
       @plugins = []
       @show_all_stack_events = false
       @parameter_strategy = Moonshot::ParameterStrategy::DefaultStrategy.new
+      @ssh_config = SSHConfig.new
     end
   end
 end

data/lib/moonshot/deployment_mechanism/code_deploy.rb

@@ -13,20 +13,33 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
   include Moonshot::CredsHelper
   include Moonshot::DoctorHelper
 
-  # @param asg [String]
+  # @param asg [Array, String]
   #   The logical name of the AutoScalingGroup to create and manage a Deployment
   #   Group for in CodeDeploy.
   # @param role [String]
   #   IAM role with AWSCodeDeployRole policy. CodeDeployRole is considered as
   #   default role if its not specified.
   # @param app_name [String, nil] (nil)
-  #   The name of the CodeDeploy Application and Deployment Group. By default,
-  #   this is the same as the stack name, and probably what you want. If you
-  #   have multiple deployments in a single Stack, they must have unique names.
-  def initialize(asg:, role: 'CodeDeployRole', app_name: nil)
-    @asg_logical_id = asg
+  #   The name of the CodeDeploy Application. By default, this is the same as
+  #   the stack name, and probably what you want. If you have multiple
+  #   deployments in a single Stack, they must have unique names.
+  # @param group_name [String, nil] (nil)
+  #   The name of the CodeDeploy Deployment Group. By default, this is the same
+  #   as app_name.
+  # @param config_name [String]
+  #   Name of the Deployment Config to use for CodeDeploy,  By default we use
+  #   CodeDeployDefault.OneAtATime.
+  def initialize(
+      asg: [],
+      role: 'CodeDeployRole',
+      app_name: nil,
+      group_name: nil,
+      config_name: 'CodeDeployDefault.OneAtATime')
+    @asg_logical_ids = asg.is_a?(Array) ? asg : [asg]
     @app_name = app_name
+    @group_name = group_name
     @codedeploy_role = role
+    @codedeploy_config = config_name
   end
 
   def post_create_hook
@@ -56,9 +69,9 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
     ilog.start_threaded 'Creating Deployment' do |s|
       res = cd_client.create_deployment(
         application_name: app_name,
-        deployment_group_name: app_name,
+        deployment_group_name: group_name,
         revision: revision_for_artifact_repo(artifact_repo, version_name),
-        deployment_config_name: 'CodeDeployDefault.OneAtATime',
+        deployment_config_name: @codedeploy_config,
         description: "Deploying version #{version_name}"
       )
       deployment_id = res.deployment_id
@@ -80,12 +93,18 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
 
   private
 
-  # By default, use the stack name as the application and deployment group
-  # names, unless one has been provided.
+  # By default, use the stack name as the application name, unless one has been
+  # provided.
   def app_name
     @app_name || stack.name
   end
 
+  # By default, use the stack name as the deployment group name, unless one has
+  # been provided.
+  def group_name
+    @group_name || stack.name
+  end
+
   def pretty_app_name
     "CodeDeploy Application #{app_name.blue}"
   end
@@ -131,27 +150,35 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
     end
   end
 
-  def auto_scaling_group
-    @auto_scaling_group ||= load_auto_scaling_group
+  def auto_scaling_groups
+    @auto_scaling_groups ||= load_auto_scaling_groups
   end
 
-  def load_auto_scaling_group
-    asg_name = stack.physical_id_for(@asg_logical_id)
-    unless asg_name
-      raise Thor::Error, "Could not find #{@asg_logical_id} resource in Stack."
-    end
+  def load_auto_scaling_groups
+    autoscaling_groups = []
+    @asg_logical_ids.each do |asg_logical_id|
+      asg_name = stack.physical_id_for(asg_logical_id)
+      unless asg_name
+        raise Thor::Error, "Could not find #{asg_logical_id} resource in Stack."
+      end
 
-    groups = as_client.describe_auto_scaling_groups(
-      auto_scaling_group_names: [asg_name])
-    if groups.auto_scaling_groups.empty?
-      raise Thor::Error, "Could not find ASG #{asg_name}."
-    end
+      groups = as_client.describe_auto_scaling_groups(
+        auto_scaling_group_names: [asg_name])
+      if groups.auto_scaling_groups.empty?
+        raise Thor::Error, "Could not find ASG #{asg_name}."
+      end
 
-    groups.auto_scaling_groups.first
+      autoscaling_groups.push(groups.auto_scaling_groups.first)
+    end
+    autoscaling_groups
   end
 
-  def asg_name
-    auto_scaling_group.auto_scaling_group_name
+  def asg_names
+    names = []
+    auto_scaling_groups.each do |auto_scaling_group|
+      names.push(auto_scaling_group.auto_scaling_group_name)
+    end
+    names
   end
 
   def application_exists?
@@ -163,13 +190,13 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
 
   def deployment_group
     cd_client.get_deployment_group(
-      application_name: app_name, deployment_group_name: app_name)
+      application_name: app_name, deployment_group_name: group_name)
              .deployment_group_info
   end
 
   def deployment_group_exists?
     cd_client.get_deployment_group(
-      application_name: app_name, deployment_group_name: app_name)
+      application_name: app_name, deployment_group_name: group_name)
     true
   rescue Aws::CodeDeploy::Errors::ApplicationDoesNotExistException,
          Aws::CodeDeploy::Errors::DeploymentGroupDoesNotExistException
@@ -178,9 +205,15 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
 
   def deployment_group_ok?
     return false unless deployment_group_exists?
-    asg = deployment_group.auto_scaling_groups.first
-    return false unless asg
-    asg.name == auto_scaling_group.auto_scaling_group_name
+    asgs = deployment_group.auto_scaling_groups
+    return false unless asgs
+    return false unless asgs.count == auto_scaling_groups.count
+    asgs.each do |asg|
+      if (auto_scaling_groups.find_index { |a| a.auto_scaling_group_name == asg.name }).nil?
+        return false
+      end
+    end
+    true
   end
 
   def role
@@ -193,7 +226,7 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
     ilog.start "Deleting #{pretty_deploy_group}." do |s|
       cd_client.delete_deployment_group(
         application_name: app_name,
-        deployment_group_name: app_name)
+        deployment_group_name: group_name)
       s.success
     end
   end
@@ -201,22 +234,28 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
   def create_deployment_group
     cd_client.create_deployment_group(
       application_name: app_name,
-      deployment_group_name: app_name,
+      deployment_group_name: group_name,
       service_role_arn: role.arn,
-      auto_scaling_groups: [asg_name])
+      auto_scaling_groups: asg_names)
   end
 
   def wait_for_asg_capacity
-    ilog.start 'Waiting for AutoScaling Group to reach capacity...' do |s|
+    ilog.start 'Waiting for AutoScaling Group(s) to reach capacity...' do |s|
       loop do
-        asg = load_auto_scaling_group
-        count = asg.instances.count { |i| i.lifecycle_state == 'InService' }
-        break if asg.desired_capacity == count
-        s.continue "DesiredCapacity is #{asg.desired_capacity}, currently #{count} instance(s) are InService." # rubocop:disable LineLength
+        asgs_at_capacity = 0
+        asgs = load_auto_scaling_groups
+        asgs.each do |asg|
+          count = asg.instances.count { |i| i.lifecycle_state == 'InService' }
+          if asg.desired_capacity == count
+            asgs_at_capacity += 1
+            s.continue "#{asg.auto_scaling_group_name} DesiredCapacity is #{asg.desired_capacity}, currently #{count} instance(s) are InService." # rubocop:disable LineLength
+          end
+        end
+        break if asgs.count == asgs_at_capacity
         sleep 5
       end
 
-      s.success 'AutoScaling Group up to capacity!'
+      s.success 'AutoScaling Group(s) up to capacity!'
     end
   end
 
@@ -239,7 +278,7 @@ class Moonshot::DeploymentMechanism::CodeDeploy # rubocop:disable ClassLength
     end
   end
 
-  def handle_deployment_failure(deployment_id) # rubocop:disable AbcSize
+  def handle_deployment_failure(deployment_id)
     instances = cd_client.list_deployment_instances(deployment_id: deployment_id)
                          .instances_list.map do |instance_id|
       cd_client.get_deployment_instance(deployment_id: deployment_id,
@@ -298,10 +337,12 @@ http://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-service-rol
   end
 
   def doctor_check_auto_scaling_resource_defined
-    if stack.template.resource_names.include?(@asg_logical_id)
-      success("Resource '#{@asg_logical_id}' exists in the CloudFormation template.") # rubocop:disable LineLength
-    else
-      critical("Resource '#{@asg_logical_id}' does not exist in the CloudFormation template!") # rubocop:disable LineLength
+    @asg_logical_ids.each do |asg_logical_id|
+      if stack.template.resource_names.include?(asg_logical_id)
+        success("Resource '#{asg_logical_id}' exists in the CloudFormation template.") # rubocop:disable LineLength
+      else
+        critical("Resource '#{asg_logical_id}' does not exist in the CloudFormation template!") # rubocop:disable LineLength
+      end
     end
   end
 end

data/lib/moonshot/shell.rb

@@ -2,7 +2,7 @@
 module Moonshot::Shell
   # Run a command, returning stdout. Stderr is suppressed unless the command
   # returns non-zero.
-  def sh_out(cmd, fail: true, stdin: '') # rubocop:disable AbcSize
+  def sh_out(cmd, fail: true, stdin: '')
     r_in, w_in = IO.pipe
     r_out, w_out = IO.pipe
     r_err, w_err = IO.pipe

data/lib/moonshot/ssh_command_builder.rb

@@ -0,0 +1,32 @@
+require 'shellwords'
+
+module Moonshot
+  # Create an ssh command from configuration.
+  class SSHCommandBuilder
+    Result = Struct.new(:cmd, :ip)
+
+    def initialize(ssh_config, instance_id)
+      @config = ssh_config
+      @instance_id = instance_id
+    end
+
+    def build(command = nil)
+      cmd = ['ssh', '-t']
+      cmd << "-i #{@config.ssh_identity_file}" if @config.ssh_identity_file
+      cmd << "-l #{@config.ssh_user}" if @config.ssh_user
+      cmd << instance_ip
+      cmd << Shellwords.escape(command) if command
+      Result.new(cmd.join(' '), instance_ip)
+    end
+
+    private
+
+    def instance_ip
+      @instance_ip ||= Aws::EC2::Client.new
+                                       .describe_instances(instance_ids: [@instance_id])
+                                       .reservations.first.instances.first.public_ip_address
+    rescue
+      raise "Failed to determine public IP address for instance #{@instance_id}!"
+    end
+  end
+end

data/lib/moonshot/ssh_config.rb

@@ -0,0 +1,6 @@
+module Moonshot
+  class SSHConfig # rubocop:disable Documentation
+    attr_accessor :ssh_identity_file
+    attr_accessor :ssh_user
+  end
+end

data/lib/moonshot/ssh_fork_executor.rb

@@ -0,0 +1,20 @@
+require 'open3'
+
+module Moonshot
+  # Run an SSH command via fork/exec.
+  class SSHForkExecutor
+    Result = Struct.new(:output, :exitstatus)
+
+    def run(cmd)
+      output = StringIO.new
+
+      exit_status = nil
+      Open3.popen3(cmd) do |_, stdout, _, wt|
+        output << stdout.read until stdout.eof?
+        exit_status = wt.value.exitstatus
+      end
+
+      Result.new(output.string.chomp, exit_status)
+    end
+  end
+end

data/lib/moonshot/ssh_target_selector.rb

@@ -0,0 +1,30 @@
+module Moonshot
+  # Choose a publically accessible instance to run commands on, given a Moonshot::Stack.
+  class SSHTargetSelector
+    def initialize(stack, asg_name: nil)
+      @asg_name = asg_name
+      @stack = stack
+    end
+
+    def choose!
+      groups = @stack.resources_of_type('AWS::AutoScaling::AutoScalingGroup')
+
+      asg = if groups.count == 1
+              groups.first
+            elsif asgs.count > 1
+              unless @asg_name
+                raise 'Multiple Auto Scaling Groups found in the stack. Please specify which '\
+                      'one to SSH into using the --auto-scaling-group (-g) option.'
+              end
+              groups.detect { |x| x.logical_resource_id == @config.ssh_auto_scaling_group_name }
+            end
+      raise 'Failed to find the Auto Scaling Group.' unless asg
+
+      Aws::AutoScaling::Client.new.describe_auto_scaling_groups(
+        auto_scaling_group_names: [asg.physical_resource_id]
+      ).auto_scaling_groups.first.instances.map(&:instance_id).first
+    rescue
+      raise 'Failed to find instances in the Auto Scaling Group!'
+    end
+  end
+end

data/lib/moonshot/stack.rb

@@ -93,18 +93,6 @@ module Moonshot
       end
     end
 
-    def ssh
-      box_id = @config.ssh_instance || instances.sort.first
-      box_ip = instance_ip(box_id)
-      cmd = ['ssh', '-t']
-      cmd << "-i #{@config.ssh_identity_file}" if @config.ssh_identity_file
-      cmd << "-l #{@config.ssh_user}" if @config.ssh_user
-      cmd << box_ip
-      cmd << @config.ssh_command if @config.ssh_command
-      puts "Opening SSH connection to #{box_id} (#{box_ip})..."
-      exec(cmd.join(' '))
-    end
-
     def parameters
       get_stack(@name)
         .parameters
@@ -188,37 +176,6 @@ module Moonshot
 
     private
 
-    def asgs
-      resources_of_type('AWS::AutoScaling::AutoScalingGroup')
-    end
-
-    def instance_ip(instance_id)
-      Aws::EC2::Client.new.describe_instances(instance_ids: [instance_id])
-                      .reservations.first.instances.first.public_ip_address
-    rescue
-      raise "Failed to determine public IP address for instance #{instance_id}."
-    end
-
-    def instances # rubocop:disable Metrics/AbcSize
-      groups = asgs
-      asg = if groups.count == 1
-              groups.first
-            elsif asgs.count > 1
-              unless @config.ssh_auto_scaling_group_name
-                raise 'Multiple Auto Scaling Groups found in the stack. Please specify which '\
-                      'one to SSH into using the --auto-scaling-group (-g) option.'
-              end
-              groups.detect { |x| x.logical_resource_id == @config.ssh_auto_scaling_group_name }
-            end
-      raise 'Failed to find the Auto Scaling Group.' unless asg
-
-      Aws::AutoScaling::Client.new.describe_auto_scaling_groups(
-        auto_scaling_group_names: [asg.physical_resource_id]
-      ).auto_scaling_groups.first.instances.map(&:instance_id)
-    rescue
-      raise 'Failed to find instances in the Auto Scaling Group.'
-    end
-
     def stack_name
       "CloudFormation Stack #{@name.blue}"
     end

data/lib/moonshot/stack_asg_printer.rb

@@ -5,7 +5,7 @@ require 'ruby-duration'
 module Moonshot
   # Display information about the AutoScaling Groups, associated ELBs, and
   # managed instances to the user.
-  class StackASGPrinter # rubocop:disable ClassLength
+  class StackASGPrinter
     include CredsHelper
 
     def initialize(stack, table)
@@ -79,7 +79,7 @@ module Moonshot
       data
     end
 
-    def add_asg_info(table, asg_info) # rubocop:disable AbcSize
+    def add_asg_info(table, asg_info)
       name = asg_info.auto_scaling_group_name.blue
       table.add_line "Name: #{name}"
 

data/lib/moonshot/stack_config.rb

@@ -4,11 +4,6 @@ module Moonshot
     attr_accessor :parent_stacks
     attr_accessor :show_all_events
     attr_accessor :parameter_strategy
-    attr_accessor :ssh_instance
-    attr_accessor :ssh_identity_file
-    attr_accessor :ssh_user
-    attr_accessor :ssh_command
-    attr_accessor :ssh_auto_scaling_group_name
 
     def initialize
       @parent_stacks = []

data/lib/moonshot/tools/asg_rollout/asg.rb

@@ -0,0 +1,164 @@
+require_relative 'asg_instance'
+require_relative 'instance_health'
+
+module Moonshot
+  module Tools
+    class ASGRollout
+      # Abstration layer with AWS Auto Scaling Groups, for the Rollout tool.
+      class ASG
+        attr_reader :name
+
+        def initialize(name)
+          @name = name
+          @last_seen_ids = []
+        end
+
+        def current_max_and_desired
+          asg = load_asg
+
+          [asg.max_size, asg.desired_capacity]
+        end
+
+        def set_max_and_desired(max, desired)
+          autoscaling.update_auto_scaling_group(
+            auto_scaling_group_name: @name,
+            max_size: max,
+            desired_capacity: desired)
+        end
+
+        def non_conforming_instances
+          asg = load_asg
+
+          asg.instances
+             .select { |i| i.launch_configuration_name != asg.launch_configuration_name }
+             .map(&:instance_id)
+        end
+
+        def wait_for_new_instance
+          # Query the ASG until an instance appears which is not in
+          # @last_seen_ids, then add it to @last_seen_ids and return
+          # it.
+          previous_ids = @last_seen_ids
+
+          loop do
+            load_asg
+
+            new_ids = @last_seen_ids - previous_ids
+            previous_ids = @last_seen_ids
+
+            unless new_ids.empty?
+              @last_seen_ids << new_ids.first
+              return new_ids.first
+            end
+
+            sleep 3
+          end
+        end
+
+        def detach_instance(id, decrement:)
+          # Store the current instance IDs for the next call to wait_for_new_instance.
+          load_asg
+
+          resp = autoscaling.detach_instances(
+            auto_scaling_group_name: @name,
+            instance_ids: [id],
+            should_decrement_desired_capacity: decrement)
+
+          activity = resp.activities.first
+          unless activity
+            raise 'Did not receive Activity from DetachInstances call!'
+          end
+
+          # Wait for the detach activity to complete:
+          loop do
+            resp = autoscaling.describe_scaling_activities(
+              auto_scaling_group_name: @name)
+
+            current_status = resp.activities
+                                 .find { |a| a.activity_id == activity.activity_id }
+                                 .status_code
+
+            case current_status
+            when 'Failed', 'Cancelled'
+              raise 'Detachment did not complete successfully!'
+            when 'Successful'
+              return
+            end
+
+            sleep 1
+          end
+        end
+
+        def instance_health(id)
+          elb_status = nil
+          elb_status = elb_instance_state(id) if elb_name
+
+          InstanceHealth.new(asg_instance_state(id), elb_status)
+        end
+
+        private
+
+        def asg_instance_state(id)
+          resp = autoscaling.describe_auto_scaling_instances(
+            instance_ids: [id])
+
+          instance_info = resp.auto_scaling_instances.first
+          return 'Missing' unless instance_info
+
+          instance_info.lifecycle_state
+        end
+
+        def elb_instance_state(id)
+          resp = loadbalancing.describe_instance_health(
+            load_balancer_name: elb_name,
+            instances: [
+              { instance_id: id }
+            ])
+
+          instance_info = resp.instance_states.first
+          unless instance_info
+            raise "Failed to call DescribeInstanceHealth for #{id}!"
+          end
+
+          instance_info.state
+        rescue Aws::ElasticLoadBalancing::Errors::InvalidInstance
+          # We expect the instance to be in an ELB, eventually.
+          'Missing'
+        end
+
+        def autoscaling
+          @autoscaling ||= Aws::AutoScaling::Client.new
+        end
+
+        def loadbalancing
+          @loadbalancing ||= Aws::ElasticLoadBalancing::Client.new
+        end
+
+        def load_asg
+          resp = autoscaling.describe_auto_scaling_groups(
+            auto_scaling_group_names: [@name])
+
+          if resp.auto_scaling_groups.empty?
+            raise "Failed to call DescribeAutoScalingGroups for #{@name}!"
+          end
+
+          asg = resp.auto_scaling_groups.first
+          @last_seen_ids = asg.instances.map(&:instance_id)
+
+          asg
+        end
+
+        def elb_name
+          return @elb_name if @elb_name
+
+          asg = load_asg
+          if asg.load_balancer_names.size > 1
+            raise 'ASGRollout does not support configurations with multiple ELBs!'
+          end
+
+          @elb_name ||= asg.load_balancer_names.first
+        end
+      end
+    end
+  end
+end

data/lib/moonshot/tools/asg_rollout/asg_instance.rb

@@ -0,0 +1,16 @@
+module Moonshot
+  module Tools
+    class ASGRollout
+      # Provides an abstraction around an Auto Scaling Group's
+      # relationship with an instance.
+      class ASGInstance
+        attr_reader :asg_name, :id
+
+        def initialize(asg_name, id, _config)
+          @asg_name = asg_name
+          @instance_id = id
+        end
+      end
+    end
+  end
+end

data/lib/moonshot/tools/asg_rollout/hook_exec_environment.rb

@@ -0,0 +1,43 @@
+require 'moonshot/ssh_fork_executor'
+
+module Moonshot
+  module Tools
+    class ASGRollout
+      # This object is passed into hooks defined in the ASGRollout
+      # process, to give them access to instances and logging
+      # facilities.
+      class HookExecEnvironment
+        attr_reader :instance_id
+
+        def initialize(config, instance_id)
+          @ilog = config.interactive_logger
+          @command_builder = Moonshot::SSHCommandBuilder.new(config.ssh_config, instance_id)
+          @instance_id = instance_id
+        end
+
+        def exec(cmd)
+          cb = @command_builder.build(cmd)
+          fe = SSHForkExecutor.new
+          fe.run(cb.cmd)
+        end
+
+        def ec2
+          Aws::EC2::Client.new
+        end
+
+        def ec2_instance
+          res = Aws::EC2::Resource.new(client: ec2)
+          res.instance(@instance_id)
+        end
+
+        def debug(msg)
+          @ilog.debug(msg)
+        end
+
+        def info(msg)
+          @ilog.info(msg)
+        end
+      end
+    end
+  end
+end

data/lib/moonshot/tools/asg_rollout/instance_health.rb

@@ -0,0 +1,36 @@
+module Moonshot
+  module Tools
+    class ASGRollout
+      class InstanceHealth # rubocop:disable Documentation
+        attr_reader :asg_status, :elb_status
+
+        VALID_ASG_IN_SERVICE_STATES = ['InService'].freeze
+        VALID_ELB_IN_SERVICE_STATES = [nil, 'InService'].freeze
+
+        VALID_ASG_OUT_OF_SERVICE_STATES = [nil, 'Missing', 'Detached'].freeze
+        VALID_ELB_OUT_OF_SERVICE_STATES = [nil, 'Missing', 'OutOfService'].freeze
+
+        def initialize(asg_status, elb_status)
+          @asg_status = asg_status
+          @elb_status = elb_status
+        end
+
+        def to_s
+          result = "ASG:#{@asg_status}"
+          result << "/ELB:#{@elb_status}" if @elb_status
+          result
+        end
+
+        def in_service?
+          VALID_ASG_IN_SERVICE_STATES.include?(@asg_status) &&
+            VALID_ELB_IN_SERVICE_STATES.include?(@elb_status)
+        end
+
+        def out_of_service?
+          VALID_ASG_OUT_OF_SERVICE_STATES.include?(@asg_status) &&
+            VALID_ELB_OUT_OF_SERVICE_STATES.include?(@elb_status)
+        end
+      end
+    end
+  end
+end

data/lib/moonshot/tools/asg_rollout.rb

@@ -0,0 +1,170 @@
+require_relative 'asg_rollout_config'
+require_relative 'asg_rollout/asg'
+require_relative 'asg_rollout/hook_exec_environment'
+
+module Moonshot
+  module Tools
+    class ASGRollout # rubocop:disable Documentation
+      attr_accessor :config
+
+      def initialize(controller:, logical_id:)
+        @config = ASGRolloutConfig.new
+        @controller = controller
+        @logical_id = logical_id
+        yield @config if block_given?
+      end
+
+      def run!
+        increase_max_and_desired
+        new_instance = wait_for_new_instance
+        wait_for_in_service(new_instance)
+
+        targets = asg.non_conforming_instances
+        last_instance = targets.last
+
+        targets.each do |instance|
+          run_pre_detach(instance) if @config.pre_detach
+          detach(instance, decrement: instance == last_instance)
+          wait_for_out_of_service(instance)
+
+          unless instance == last_instance
+            new_instance = wait_for_new_instance
+            wait_for_in_service(new_instance)
+          end
+
+          wait_for_terminate_when_hook(instance) if @config.terminate_when
+          terminate(instance)
+        end
+      ensure
+        log.start_threaded 'Restoring MaxSize/DesiredCapacity values to normal...' do |s|
+          asg.set_max_and_desired(@max, @desired)
+
+          s.success 'Restored MaxSize/DesiredCapacity values to normal!'
+        end
+      end
+
+      private
+
+      def increase_max_and_desired
+        log.start_threaded 'Increasing MaxSize/DesiredCapacity by 1.' do |s|
+          @max, @desired = asg.current_max_and_desired
+
+          asg.set_max_and_desired(@max + 1, @desired + 1)
+          s.success 'Increased MaxSize/DesiredCapacity by 1.'
+        end
+      end
+
+      def wait_for_new_instance
+        new_instance = nil
+        log.start_threaded 'Waiting for a new instance to join Auto Scaling Group...' do |s|
+          new_instance = asg.wait_for_new_instance
+          s.success "A wild #{new_instance.blue} appears!"
+        end
+        new_instance
+      end
+
+      def wait_for_in_service(new_instance)
+        log.start_threaded "Waiting for #{new_instance.blue} to be InService..." do |s|
+          instance_health = nil
+
+          loop do
+            instance_health = asg.instance_health(new_instance)
+            break if instance_health.in_service?
+
+            s.continue "Instance #{new_instance.blue} is #{instance_health}..."
+
+            sleep @config.instance_health_delay
+          end
+
+          s.success "Instance #{new_instance.blue} is #{instance_health}!"
+        end
+      end
+
+      def run_pre_detach(instance)
+        if @config.pre_detach
+          log.start_threaded "Running PreDetach hook on #{instance.blue}..." do |s|
+            he = HookExecEnvironment.new(@controller.config, instance)
+            if false == @config.pre_detach.call(he)
+              s.failure "PreDetach hook failed for #{instance.blue}!"
+              raise "PreDetach hook failed for #{instance.blue}!"
+            end
+
+            s.success "PreDetach hook complete for #{instance.blue}!"
+          end
+        end
+      end
+
+      def detach(instance, decrement:)
+        log.start_threaded "Detaching instance #{instance.blue}..." do |s|
+          asg.detach_instance(instance, decrement: decrement)
+
+          if decrement
+            s.success "Detached instance #{instance.blue}, and decremented DesiredCapacity."
+          else
+            s.success "Detached instance #{instance.blue}."
+          end
+        end
+      end
+
+      def wait_for_out_of_service(instance)
+        log.start_threaded "Waiting for #{instance.blue} to be OutOfService..." do |s|
+          instance_health = nil
+
+          loop do
+            instance_health = asg.instance_health(instance)
+            break if instance_health.out_of_service?
+
+            s.continue "Instance #{instance.blue} is #{instance_health}..."
+
+            sleep @config.instance_health_delay
+          end
+
+          s.success "Instance #{instance.blue} is #{instance_health}!"
+        end
+      end
+
+      def wait_for_terminate_when_hook(instance)
+        log.start_threaded "Waiting for TerminateWhen hook for #{instance.blue}..." do |s|
+          start = Time.now.to_f
+          he = HookExecEnvironment.new(@controller.config, instance)
+          timeout = @config.terminate_when_timeout
+
+          loop do
+            break if @config.terminate_when.call(he)
+            sleep @config.terminate_when_delay
+
+            if Time.now.to_f - start > timeout
+              s.failure "TerminateWhen for #{instance.blue} did not complete in #{timeout} seconds!"
+              raise "TerminateWhen for #{instance.blue} did not complete in #{timeout} seconds!"
+            end
+          end
+
+          s.success "Completed TerminateWhen check for #{instance.blue}!"
+        end
+      end
+
+      def terminate(instance)
+        log.start_threaded "Terminating #{instance.blue}..." do |s|
+          he = HookExecEnvironment.new(@controller.config, instance)
+          @config.terminate.call(he)
+          s.success "Terminated #{instance.blue}!"
+        end
+      end
+
+      def asg
+        return @asg if @asg
+
+        asg_name = @controller.stack.physical_id_for(@logical_id)
+        unless asg_name
+          raise "Could not find Auto Scaling Group #{@logical_id}!"
+        end
+
+        @asg ||= ASGRollout::ASG.new(asg_name)
+      end
+
+      def log
+        @controller.config.interactive_logger
+      end
+    end
+  end
+end

data/lib/moonshot/tools/asg_rollout_config.rb

@@ -0,0 +1,39 @@
+module Moonshot
+  module Tools
+    class ASGRolloutConfig # rubocop:disable Documentation
+      attr_reader :pre_detach, :terminate_when, :terminate_when_timeout, :terminate
+      attr_accessor :terminate_when_delay, :instance_health_delay
+
+      def initialize
+        @instance_health_delay = 2
+        @terminate_when_delay = 1
+        @terminate_when_timeout = 300
+        @terminate = proc do |h|
+          h.ec2_instance.terminate
+        end
+      end
+
+      def pre_detach=(value)
+        raise ArgumentError, 'pre_detach must be callable' unless value.respond_to?(:call)
+
+        @pre_detach = value
+      end
+
+      def terminate_when=(value)
+        raise ArgumentError, 'terminate_when must be callable' unless value.respond_to?(:call)
+
+        @terminate_when = value
+      end
+
+      def terminate_when_timeout=(value)
+        @terminate_when_timeout = Float(value)
+      end
+
+      def terminate=(value)
+        raise ArgumentError, 'terminate must be callable' unless value.respond_to?(:call)
+
+        @terminate = value
+      end
+    end
+  end
+end

data/lib/moonshot.rb

@@ -39,7 +39,10 @@ end
   'build_mechanism/github_release',
   'build_mechanism/travis_deploy',
   'build_mechanism/version_proxy',
-  'deployment_mechanism/code_deploy'
+  'deployment_mechanism/code_deploy',
+
+  # Core Tools
+  'tools/asg_rollout'
 ].each { |f| require_relative "moonshot/#{f}" }
 
 # Bundled plugins

data/lib/plugins/backup.rb

@@ -5,7 +5,7 @@ require_relative '../moonshot/creds_helper'
 module Moonshot
   module Plugins
     # Moonshot plugin class for deflating and uploading files on given hooks
-    class Backup # rubocop:disable Metrics/ClassLength
+    class Backup
       include Moonshot::CredsHelper
 
       attr_accessor :bucket,
@@ -42,7 +42,7 @@ module Moonshot
       # to an S3 bucket.
       #
       # @param resources [Resources] injected Moonshot resources
-      def backup(resources) # rubocop:disable Metrics/AbcSize
+      def backup(resources)
         raise ArgumentError if resources.nil?
 
         @app_name = resources.stack.app_name

metadata

@@ -1,20 +1,23 @@
 --- !ruby/object:Gem::Specification
 name: moonshot
 version: !ruby/object:Gem::Version
-  version: 0.7.6
+  version: 0.7.7
 platform: ruby
 authors:
 - Cloud Engineering <engineering@acquia.com>
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-11 00:00:00.000000000 Z
+date: 2016-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.0
   type: :runtime
@@ -22,6 +25,9 @@ dependencies:
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.0
 - !ruby/object:Gem::Dependency
@@ -58,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.1.1
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
   name: rotp
   requirement: !ruby/object:Gem::Requirement
@@ -136,6 +142,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: travis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: vandamme
   requirement: !ruby/object:Gem::Requirement
@@ -218,6 +238,10 @@ files:
 - lib/moonshot/resources.rb
 - lib/moonshot/resources_helper.rb
 - lib/moonshot/shell.rb
+- lib/moonshot/ssh_command_builder.rb
+- lib/moonshot/ssh_config.rb
+- lib/moonshot/ssh_fork_executor.rb
+- lib/moonshot/ssh_target_selector.rb
 - lib/moonshot/stack.rb
 - lib/moonshot/stack_asg_printer.rb
 - lib/moonshot/stack_config.rb
@@ -226,6 +250,12 @@ files:
 - lib/moonshot/stack_output_printer.rb
 - lib/moonshot/stack_parameter_printer.rb
 - lib/moonshot/stack_template.rb
+- lib/moonshot/tools/asg_rollout.rb
+- lib/moonshot/tools/asg_rollout/asg.rb
+- lib/moonshot/tools/asg_rollout/asg_instance.rb
+- lib/moonshot/tools/asg_rollout/hook_exec_environment.rb
+- lib/moonshot/tools/asg_rollout/instance_health.rb
+- lib/moonshot/tools/asg_rollout_config.rb
 - lib/moonshot/unicode_table.rb
 - lib/plugins/backup.rb
 homepage: https://github.com/acquia/moonshot