aerosol 0.5.1

data/lib/aerosol/launch_configuration.rb

@@ -0,0 +1,106 @@
+class Aerosol::LaunchConfiguration
+  include Aerosol::AWSModel
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[aerosol launch_configuration]'
+  aws_attribute :aws_identifier              => 'LaunchConfigurationName',
+                :ami                         => 'ImageId',
+                :instance_type               => 'InstanceType',
+                :security_groups             => 'SecurityGroups',
+                :user_data                   => 'UserData',
+                :iam_role                    => 'IamInstanceProfile',
+                :kernel_id                   => 'KernelId',
+                :key_name                    => 'KeyName',
+                :spot_price                  => 'SpotPrice',
+                :created_time                => 'CreatedTime',
+                :associate_public_ip_address => 'AssociatePublicIpAddress'
+
+  primary_key :aws_identifier
+  default_value(:security_groups) { [] }
+
+  def aws_identifier(arg = nil)
+    if arg
+      raise "You cannot set the aws_identifer directly" unless from_aws
+      @aws_identifier = arg
+    else
+      @aws_identifier || default_identifier
+    end
+  end
+
+  def security_group(group)
+    security_groups << group
+  end
+
+  def create!
+    ensure_present! :ami, :instance_type
+
+    info self.to_s
+    conn.create_launch_configuration(ami, instance_type, aws_identifier, create_options)
+    sleep 10 # TODO: switch to fog models and .wait_for { ready? }
+  end
+
+  def destroy!
+    info self.to_s
+    conn.delete_launch_configuration(aws_identifier)
+  end
+
+  def all_instances
+    Aerosol::Instance.all.select { |instance|
+      !instance.launch_configuration.nil? &&
+        (instance.launch_configuration.aws_identifier == self.aws_identifier)
+    }.each(&:description)
+  end
+
+  def self.request_all_for_token(next_token)
+    options = next_token.nil? ? {} : { 'NextToken' => next_token }
+    Aerosol::AWS.auto_scaling
+                .describe_launch_configurations(options)
+                .body
+                .[]('DescribeLaunchConfigurationsResult')
+  end
+
+  def self.request_all
+    next_token = nil
+    lcs = []
+
+    begin
+      new_lcs = request_all_for_token(next_token)
+      lcs.concat(new_lcs['LaunchConfigurations'])
+      next_token = new_lcs['NextToken']
+    end while !next_token.nil?
+
+    lcs
+  end
+
+  def to_s
+    %{Aerosol::LaunchConfiguration { \
+"aws_identifier" => "#{aws_identifier}", \
+"ami" => "#{ami}", \
+"instance_type" => "#{instance_type}", \
+"security_groups" => #{security_groups.to_s}, \
+"user_data" => "#{user_data}", \
+"iam_role" => "#{iam_role}", \
+"kernel_id" => "#{kernel_id}", \
+"key_name" => "#{key_name}", \
+"spot_price" => "#{spot_price}", \
+"created_time" => "#{created_time}" \
+}}
+  end
+
+private
+  def create_options
+    { # TODO Add dsl so that 'BlockDeviceMappings' may be specified
+      'IamInstanceProfile' => iam_role,
+      'KernelId' => kernel_id,
+      'KeyName' => key_name,
+      'SecurityGroups' => security_groups,
+      'SpotPrice' => spot_price,
+      'UserData' => Aerosol::Util.strip_heredoc(user_data || ''),
+      'AssociatePublicIpAddress' => associate_public_ip_address
+    }.reject { |k, v| v.nil? }
+  end
+
+  def conn
+    Aerosol::AWS.auto_scaling
+  end
+end

data/lib/aerosol/rake_task.rb

@@ -0,0 +1,141 @@
+require 'rake'
+require 'aerosol'
+
+$rake_task_logger = Dockly::Util::Logger.new('[aerosol rake_task]', STDOUT, false)
+
+class Rake::AutoScalingTask < Rake::Task
+  def needed?
+    !auto_scaling.exists?
+  end
+
+  def auto_scaling
+    Aerosol.auto_scaling(name.split(':').last.to_sym)
+  end
+end
+
+module Rake::DSL
+  def auto_scaling(*args, &block)
+    Rake::AutoScalingTask.define_task(*args, &block)
+  end
+end
+
+namespace :aerosol do
+  desc "Verify an aerosol.rb file exists"
+  task :load do
+    raise "No aerosol.rb found!" unless File.exist?('aerosol.rb')
+  end
+
+  namespace :auto_scaling do
+    Aerosol.auto_scalings.values.reject(&:from_aws).each do |inst|
+      auto_scaling inst.name => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        inst.create
+      end
+    end
+  end
+
+  namespace :ssh do
+    Aerosol.deploys.values.each do |inst|
+      desc "Prints out ssh command to all instances of the latest deploy of #{inst.name}"
+      task inst.name do |name|
+        Thread.current[:rake_task] = name
+        inst.generate_ssh_commands.each do |ssh_command|
+          puts ssh_command
+        end
+      end
+    end
+  end
+
+  all_deploy_tasks = []
+  all_asynch_deploy_tasks = []
+
+  namespace :env do
+    Aerosol.envs.values.each do |env|
+      desc "Run all of the deploys for #{env.name} in parallel"
+      multitask env.name => env.deploy.map { |dep| "aerosol:#{dep.name}:all" }
+    end
+  end
+
+
+  Aerosol.deploys.values.each do |inst|
+    namespace :"#{inst.name}" do
+      desc "Runs the ActiveRecord migration through the SSH connection given"
+      task :run_migration => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        Aerosol::Runner.new.with_deploy(inst.name) do |runner|
+          runner.run_migration
+        end
+      end
+
+      desc "Creates a new auto scaling group for the current git hash"
+      task :create_auto_scaling_group => "aerosol:auto_scaling:#{inst.auto_scaling.name}"
+
+      desc "Waits for instances of the new autoscaling groups to start up"
+      task :wait_for_new_instances => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        Aerosol::Runner.new.with_deploy(inst.name) do |runner|
+          runner.wait_for_new_instances
+        end
+      end
+
+      desc "Runs command to shut down the application on the old instances instead of just terminating"
+      task :stop_old_app => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        Aerosol::Runner.new.with_deploy(inst.name) do |runner|
+          runner.stop_app
+        end
+      end
+
+      desc "Terminates instances with the current tag and different git hash"
+      task :destroy_old_auto_scaling_groups => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        Aerosol::Runner.new.with_deploy(inst.name) do |runner|
+          runner.destroy_old_auto_scaling_groups
+        end
+      end
+
+      desc "Terminates instances with the current tag and current git hash"
+      task :destroy_new_auto_scaling_groups => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        Aerosol::Runner.new.with_deploy(inst.name) do |runner|
+          runner.destroy_new_auto_scaling_groups
+        end
+      end
+
+      desc "Runs a post deploy command"
+      task :run_post_deploy => 'aerosol:load' do |name|
+        Thread.current[:rake_task] = name
+        inst.run_post_deploy
+      end
+
+      ##
+
+      desc "Runs migration and creates auto scaling groups"
+      task :all_prep => [:run_migration, :create_auto_scaling_group]
+
+      desc "Waits for new instances, stops old application, destroys old auto scaling groups "\
+           "and runs the post deploy command"
+      task :all_release => [:wait_for_new_instances, :stop_old_app, :destroy_old_auto_scaling_groups, :run_post_deploy]
+
+      desc "Run migration, create auto scaling group, wait for instances, stop old application, "\
+           "destroy old auto scaling groups and run the post deploy command"
+      task :all => [:all_prep, :all_release]
+      all_deploy_tasks << "aerosol:#{inst.name}:all"
+
+      ##
+
+      desc "Runs migration and creates auto scaling groups in parallel"
+      multitask :all_asynch_prep => [:run_migration, :create_auto_scaling_group]
+
+      desc "Same as `all` but runs the migration and creates auto scaling groups in parallel"
+      task :all_asynch => [:all_asynch_prep, :all_release]
+      all_asynch_deploy_tasks << "aerosol:#{inst.name}:all_asynch"
+    end
+  end
+
+  desc "Runs all the all deploy tasks in the aerosol.rb"
+  task :deploy_all => all_deploy_tasks
+
+  desc "Runs all the all deploy tasks in the aerosol.rb in parallel"
+  multitask :deploy_all_asynch => all_asynch_deploy_tasks
+end

data/lib/aerosol/runner.rb

@@ -0,0 +1,329 @@
+require 'socket'
+require 'active_record'
+require 'grit'
+require 'timeout'
+
+class Aerosol::Runner
+  extend Dockly::Util::Delegate
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[aerosol runner]'
+  attr_reader :deploy, :log_pids
+
+  def initialize
+    @log_pids = {}
+  end
+
+  def run_migration
+    require_deploy!
+    return unless deploy.migrate?
+    raise 'To run a migration, $RAILS_ENV must be set.' if ENV['RAILS_ENV'].nil?
+
+    info "running migration"
+    begin
+      info "loading config for env #{ENV['RAILS_ENV']}"
+      original_config = YAML.load(ERB.new(File.read(db_config_path)).result)[ENV['RAILS_ENV']]
+      debug "creating ssh tunnel"
+      migration_ssh.with_connection do |session|
+        # session.logger.sev_threshold=Logger::Severity::DEBUG
+        debug "finding free port"
+        port = random_open_port
+        db_port = original_config['port'] || 3306 # TODO: get default port from DB driver
+        host = original_config['host']
+        info "forwarding 127.0.0.1:#{port} --> #{host}:#{db_port}"
+        session.forward.local(port, host, db_port)
+        child = fork do
+          GC.disable
+          with_prefix('child:') do |logger|
+            logger.debug "establishing connection"
+            ActiveRecord::Base.establish_connection(original_config.merge(
+                                                      'host' => '127.0.0.1',
+                                                      'port' => port
+                                                    ))
+            logger.info "running migration"
+            ActiveRecord::Migrator.migrate(%w[db/migrate])
+          end
+        end
+        debug "waiting for child"
+        exitstatus = nil
+        session.loop(0.1) do
+          pid = Process.waitpid(child, Process::WNOHANG)
+          exitstatus = $?.exitstatus if !pid.nil?
+          pid.nil?
+        end
+        raise "migration failed: #{exitstatus}" unless exitstatus == 0
+      end
+      info "complete"
+    ensure
+      ActiveRecord::Base.clear_all_connections!
+    end
+    info "migration ran"
+  end
+
+  def wait_for_new_instances
+    require_deploy!
+    info "waiting for new instances"
+
+    live_instances = []
+    Timeout.timeout(instance_live_grace_period) do
+      loop do
+        current_instances = new_instances
+        remaining_instances = current_instances - live_instances
+        info "waiting for instances to be live (#{remaining_instances.count} remaining)"
+        debug "current instances: #{current_instances.map(&:id)}"
+        debug "live instances: #{live_instances.map(&:id)}"
+        live_instances.concat(remaining_instances.select { |instance| healthy?(instance) })
+        break if (current_instances - live_instances).empty?
+        debug 'sleeping for 10 seconds'
+        sleep(10)
+      end
+    end
+
+    info 'new instances are up'
+  rescue Timeout::Error
+    raise "[aerosol runner] site live check timed out after #{instance_live_grace_period} seconds"
+  ensure
+    log_pids.each do |instance_id, fork|
+      debug "Killing tailing for #{instance_id}: #{Time.now}"
+      Process.kill('HUP', fork)
+      debug "Killed process for #{instance_id}: #{Time.now}"
+      debug "Waiting for process to die"
+      Process.wait(fork)
+      debug "Process ended for #{instance_id}: #{Time.now}"
+    end
+  end
+
+  def healthy?(instance)
+    debug "Checking if #{instance.id} is healthy"
+
+    unless instance.live?
+      debug "#{instance.id} is not live"
+      return false
+    end
+
+    debug "trying to SSH to #{instance.id}"
+    success = false
+    ssh.with_connection(instance) do |session|
+      start_tailing_logs(ssh, instance) if log_pids[instance.id].nil?
+      debug "checking if #{instance.id} is healthy"
+      success = if is_alive?.nil?
+        debug 'Using default site live check'
+        check_site_live(session)
+      else
+        debug 'Using custom site live check'
+        is_alive?.call(session, self)
+      end
+    end
+
+    if success
+      debug "#{instance.id} is healthy"
+    else
+      debug "#{instance.id} is not healthy"
+    end
+    success
+  rescue => ex
+    debug "#{instance.id} is not healthy: #{ex.message}"
+    false
+  end
+
+  def check_site_live(session)
+    command = [
+      'wget',
+      '-q',
+      # Since we're hitting localhost, the cert will always be invalid, so don't try to check it.
+      deploy.ssl ? '--no-check-certificate' : nil,
+      "'#{deploy.live_check_url}'",
+      '-O',
+      '/dev/null'
+    ].compact.join(' ')
+
+    debug "running #{command}"
+    ret = ssh_exec!(session, command)
+    debug "finished running #{command}"
+    ret[:exit_status].zero?
+  end
+
+  def start_tailing_logs(ssh, instance)
+    if tail_logs && log_files.length > 0
+      command = [
+        'sudo', 'tail', '-f', *log_files
+      ].join(' ')
+
+      log_pids[instance.id] ||= ssh_fork(command, ssh, instance)
+    end
+  end
+
+  def ssh_fork(command, ssh, instance)
+    debug 'starting ssh fork'
+    fork do
+      Signal.trap('HUP') do
+        debug 'Killing tailing session'
+        Process.exit!
+      end
+      debug 'starting tail'
+      begin
+        ssh.with_connection(instance) do |session|
+          debug 'tailing session connected'
+          buffer = ''
+          ssh_exec!(session, command) do |stream, data|
+            data.lines.each do |line|
+              if line.end_with?($/)
+                debug "[#{instance.id}] #{stream}: #{buffer + line}"
+                buffer = ''
+              else
+                buffer = line
+              end
+            end
+          end
+        end
+      rescue => ex
+        error "#{ex.class}: #{ex.message}"
+        error "#{ex.backtrace.join("\n")}"
+      ensure
+        debug 'finished'
+      end
+    end
+  end
+
+  def stop_app
+    info "stopping old app"
+    to_stop = old_instances
+
+    info "starting with #{to_stop.length} instances to stop"
+
+    stop_app_retries.succ.times do |n|
+      break if to_stop.empty?
+      debug "stop app: #{to_stop.length} instances remaining"
+      to_stop.reject! { |instance| stop_one_app(instance) }
+    end
+
+    if to_stop.length.zero?
+      info "successfully stopped the app on each old instance"
+    elsif !continue_if_stop_app_fails
+      raise "Failed to stop app on #{to_stop.length} instances"
+    end
+    info "stopped old app"
+  end
+
+  def destroy_old_auto_scaling_groups
+    require_deploy!
+    info "destroying old autoscaling groups"
+    sleep deploy.sleep_before_termination
+    old_auto_scaling_groups.map(&:destroy)
+    info "destroyed old autoscaling groups"
+  end
+
+  def destroy_new_auto_scaling_groups
+    require_deploy!
+    info "destroying autoscaling groups created for this sha"
+    new_auto_scaling_groups.map(&:destroy)
+    info "destroyed new autoscaling groups"
+  end
+
+  def old_instances
+    require_deploy!
+    old_auto_scaling_groups.map(&:launch_configuration).compact.map(&:all_instances).flatten.compact
+  end
+
+  def old_auto_scaling_groups
+    select_auto_scaling_groups { |asg| asg.tags['GitSha'] != auto_scaling.tags['GitSha'] }
+  end
+
+  def new_auto_scaling_groups
+    select_auto_scaling_groups { |asg| asg.tags['GitSha'] == auto_scaling.tags['GitSha'] }
+  end
+
+  def select_auto_scaling_groups(&block)
+    require_deploy!
+    Aerosol::LaunchConfiguration.all # load all of the launch configurations first
+    Aerosol::AutoScaling.all.select { |asg|
+      (asg.tags['Deploy'].to_s == auto_scaling.tags['Deploy']) &&
+        (block.nil? ? true : block.call(asg))
+    }
+  end
+
+  def new_instances
+    require_deploy!
+    while launch_configuration.all_instances.length < auto_scaling.min_size
+      info "Waiting for instances to come up"
+      sleep 10
+    end
+    launch_configuration.all_instances
+  end
+
+  def with_deploy(name)
+    unless dep = Aerosol::Deploy[name]
+      raise "No deploy named '#{name}'"
+    end
+    original = @deploy
+    @deploy = dep
+    yield self
+    @deploy = original
+  end
+
+  def require_deploy!
+    raise "@deploy must be present" if deploy.nil?
+  end
+
+  def git_sha
+    @git_sha ||= Aerosol::Util.git_sha
+  end
+
+  delegate :ssh, :migration_ssh, :package, :auto_scaling, :stop_command,
+           :live_check, :db_config_path, :instance_live_grace_period,
+           :app_port, :continue_if_stop_app_fails, :stop_app_retries,
+           :is_alive?, :log_files, :tail_logs, :to => :deploy
+  delegate :launch_configuration, :to => :auto_scaling
+
+private
+
+  def stop_one_app(instance)
+    debug "attempting to stop app on: #{instance.public_hostname}"
+    ssh.with_connection(instance) do |session|
+      session.exec!(stop_command)
+      session.loop
+    end
+    info "successfully stopped app on: #{instance.public_hostname}"
+    true
+  rescue => ex
+    warn "stop app failed on #{instance.public_hostname} due to: #{ex}"
+    false
+  end
+
+  # inspired by: http://stackoverflow.com/questions/3386233/how-to-get-exit-status-with-rubys-netssh-library
+  def ssh_exec!(ssh, command, options = {}, &block)
+    res = { :out => "", :err => "", :exit_status => nil }
+    ssh.open_channel do |channel|
+      if options[:tty]
+        channel.request_pty do |ch, success|
+          raise "could not start a pseudo-tty" unless success
+          channel = ch
+        end
+      end
+
+      channel.exec(command) do |ch, success|
+        raise "unable to run remote cmd: #{command}" unless success
+
+        channel.on_data do |_, data|
+          block.call(:out, data) unless block.nil?
+          res[:out] << data
+        end
+        channel.on_extended_data do |_, type, data|
+          block.call(:err, data) unless block.nil?
+          res[:err] << data
+        end
+        channel.on_request("exit-status") { |_, data| res[:exit_status] = data.read_long }
+      end
+    end
+    ssh.loop
+    res
+  end
+
+  def random_open_port
+    socket = Socket.new(:INET, :STREAM, 0)
+    socket.bind(Addrinfo.tcp("127.0.0.1", 0))
+    port = socket.local_address.ip_port
+    socket.close
+    port
+  end
+end

data/lib/aerosol/util.rb

@@ -0,0 +1,41 @@
+require 'grit'
+
+module Aerosol::Util
+  extend self
+
+  def is_tar?(path)
+    if File.size(path) < 262
+      return false
+    end
+    magic = nil
+    File.open(path, "r") do |f|
+      f.read(257)
+      magic = f.read(5)
+    end
+    magic == "ustar"
+  end
+
+  def is_gzip?(path)
+    if File.size(path) < 2
+      return false
+    end
+    magic = nil
+    File.open(path, "r") do |f|
+      magic = f.read(2)
+    end
+    magic = magic.unpack('H*')[0]
+    magic == "1f8b"
+  end
+
+  def strip_heredoc(str)
+    str.gsub(/^#{str[/\A\s*/]}/, '')
+  end
+
+  def git_repo
+    @git_repo ||= Grit::Repo.new('.')
+  end
+
+  def git_sha
+    @git_sha ||= git_repo.git.show.lines.first.chomp.match(/^commit ([a-f0-9]+)$/)[1][0..6] rescue 'unknown'
+  end
+end

data/lib/aerosol/version.rb

@@ -0,0 +1,5 @@
+# Copyright Swipely, Inc.  All rights reserved.
+
+module Aerosol
+  VERSION = '0.5.1'
+end

data/lib/aerosol.rb

@@ -0,0 +1,83 @@
+require 'fog'
+require 'dockly/util'
+
+module Aerosol
+  require 'aerosol/aws'
+  require 'aerosol/util'
+  require 'aerosol/aws_model'
+  require 'aerosol/launch_configuration'
+  require 'aerosol/auto_scaling'
+  require 'aerosol/instance'
+  require 'aerosol/connection'
+  require 'aerosol/deploy'
+  require 'aerosol/env'
+
+  attr_reader :deploy, :instance, :git_sha, :namespace
+  attr_writer :load_file
+
+  LOAD_FILE = 'aerosol.rb'
+
+  def load_file
+    @load_file || LOAD_FILE
+  end
+
+  def inst
+    @instance ||= load_inst
+  end
+
+  def load_inst
+    setup.tap do |state|
+      if File.exists?(load_file)
+        instance_eval(IO.read(load_file), load_file)
+      end
+    end
+  end
+
+  def namespace(value = nil)
+    if value.nil?
+      @namespace
+    else
+      @namespace = value
+    end
+  end
+
+  def setup
+    {
+      :auto_scalings => Aerosol::AutoScaling.instances,
+      :deploys => Aerosol::Deploy.instances,
+      :launch_configurations => Aerosol::LaunchConfiguration.instances,
+      :sshs => Aerosol::Connection.instances,
+      :envs => Aerosol::Env.instances
+    }
+  end
+
+  {
+    :auto_scaling => Aerosol::AutoScaling,
+    :deploy => Aerosol::Deploy,
+    :launch_configuration => Aerosol::LaunchConfiguration,
+    :ssh => Aerosol::Connection,
+    :env => Aerosol::Env
+  }.each do |method, klass|
+    define_method(method) do |sym, &block|
+      if block.nil?
+        inst[:"#{method}s"][sym]
+      else
+        klass.new!(:name => sym, &block)
+      end
+    end
+  end
+
+  [:auto_scalings, :deploys, :launch_configurations, :sshs, :envs].each do |method|
+    define_method(method) do
+      inst[method]
+    end
+  end
+
+  module_function :inst, :load_inst, :setup, :load_file, :load_file=,
+                  :auto_scaling,  :launch_configuration,  :deploy,  :ssh, :git_sha,
+                  :auto_scalings, :launch_configurations, :deploys, :sshs,
+                  :namespace, :env, :envs
+end
+
+require 'aerosol/runner'
+require 'aerosol/rake_task'