aerosol 0.5.1

data/lib/aerosol/auto_scaling.rb

@@ -0,0 +1,240 @@
+class Aerosol::AutoScaling
+  include Aerosol::AWSModel
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[aerosol auto_scaling]'
+  aws_attribute :aws_identifier            => 'AutoScalingGroupName',
+                :availability_zones        => 'AvailabilityZones',
+                :min_size                  => 'MinSize',
+                :max_size                  => 'MaxSize',
+                :default_cooldown          => 'DefaultCooldown',
+                :desired_capacity          => 'DesiredCapacity',
+                :health_check_grace_period => 'HealthCheckGracePeriod',
+                :health_check_type         => 'HealthCheckType',
+                :load_balancer_names       => 'LoadBalancerNames',
+                :placement_group           => 'PlacementGroup',
+                :tag_from_array            => 'Tags',
+                :created_time              => 'CreatedTime',
+                :vpc_zone_identifier       => 'VPCZoneIdentifier'
+  aws_class_attribute :launch_configuration, Aerosol::LaunchConfiguration
+  primary_key :aws_identifier
+
+  def initialize(options={}, &block)
+    tag = options.delete(:tag)
+    super(options, &block)
+
+    tags.merge!(tag) unless tag.nil?
+
+    tags["GitSha"] ||= Aerosol::Util.git_sha
+    tags["Deploy"] ||= namespaced_name
+  end
+
+  def aws_identifier(arg = nil)
+    if arg
+      raise "You cannot set the aws_identifer directly" unless from_aws
+      @aws_identifier = arg
+    else
+      @aws_identifier || default_identifier
+    end
+  end
+
+  def exists?
+    info "auto_scaling: needed?: #{namespaced_name}: " +
+         "checking for auto scaling group: #{aws_identifier}"
+    exists = super
+    info "auto scaling: needed?: #{namespaced_name}: " +
+         "#{exists ? 'found' : 'did not find'} auto scaling group: #{aws_identifier}"
+    exists
+  end
+
+  def create!
+    ensure_present! :launch_configuration, :max_size, :min_size
+    raise 'availability_zones or vpc_zone_identifier must be set' if [availability_zones, vpc_zone_identifier].none?
+
+    info "creating auto scaling group"
+    launch_configuration.create
+    info self.inspect
+
+    conn.create_auto_scaling_group(aws_identifier, availability_zones,
+                                  launch_configuration.aws_identifier,
+                                  max_size, min_size,
+                                  create_options)
+    make_fake_instances
+    sleep 10 # TODO: switch to fog models and .wait_for { ready? }
+  end
+
+  def destroy!
+    info self.inspect
+    conn.delete_auto_scaling_group(aws_identifier, 'ForceDelete' => true)
+    begin
+      (0..2).each { break if deleting?; sleep 1 }
+      launch_configuration.destroy
+    rescue => ex
+      info "Launch Config: #{launch_configuration} for #{aws_identifier} was not deleted."
+      info ex.message
+    end
+  end
+
+  def deleting?
+    asgs = conn.describe_auto_scaling_groups("AutoScalingGroupNames" => self.aws_identifier)
+        .body
+        .[]('DescribeAutoScalingGroupsResult')
+        .[]('AutoScalingGroups')
+
+    return true if asgs.empty?
+
+    asgs.first['Status'].to_s.include?('Delete')
+  end
+
+  def all_instances
+    Aerosol::AWS.auto_scaling
+                .describe_auto_scaling_groups('AutoScalingGroupNames' => self.aws_identifier)
+                .body
+                .[]('DescribeAutoScalingGroupsResult')
+                .[]('AutoScalingGroups')
+                .first
+                .[]('Instances')
+                .map { |instance| Aerosol::Instance.from_hash(instance) }
+  end
+
+  def tag(val)
+    tags.merge!(val)
+  end
+
+  def tags
+    @tags ||= {}
+  end
+
+  def self.request_all_for_token(next_token)
+    options = next_token.nil? ? {} : { 'NexToken' => '' }
+    Aerosol::AWS.auto_scaling
+                .describe_auto_scaling_groups(options)
+                .body
+                .[]('DescribeAutoScalingGroupsResult')
+  end
+
+  def self.request_all
+    next_token = nil
+    asgs = []
+
+    begin
+      new_asgs = request_all_for_token(next_token)
+      asgs.concat(new_asgs['AutoScalingGroups'])
+      next_token = new_asgs['NextToken']
+    end while !next_token.nil?
+
+    asgs
+  end
+
+  def self.latest_for_tag(key, value)
+    all.select  { |group| group.tags[key] == value }
+       .sort_by { |group| group.created_time }.last
+  end
+
+  def to_s
+    %{Aerosol::AutoScaling { \
+"aws_identifier" => "#{aws_identifier}", \
+"availability_zones" => "#{availability_zones}", \
+"min_size" => "#{min_size}", \
+"max_size" => "#{max_size}", \
+"default_cooldown" => "#{default_cooldown}", \
+"desired_capacity" => "#{desired_capacity}", \
+"health_check_grace_period" => "#{health_check_grace_period}", \
+"health_check_type" => "#{health_check_type}", \
+"load_balancer_names" => "#{load_balancer_names}", \
+"placement_group" => "#{placement_group}", \
+"tags" => #{tags.to_s}, \
+"created_time" => "#{created_time}" \
+}}
+  end
+
+private
+  def conn
+    Aerosol::AWS.auto_scaling
+  end
+
+  def create_options
+    {
+      'DefaultCooldown' => default_cooldown,
+      'DesiredCapacity' => desired_capacity,
+      'HealthCheckGracePeriod' => health_check_grace_period,
+      'HealthCheckType' => health_check_type,
+      'LoadBalancerNames' => load_balancer_names,
+      'PlacementGroup' => placement_group,
+      'Tags' => tags,
+      'VPCZoneIdentifier' => vpc_zone_identifier
+    }.reject { |k, v| v.nil? }
+  end
+
+  def tag_from_array(ary)
+    if ary.is_a? Hash
+      ary.each do |key, value|
+        tag key => value
+      end
+    else
+      ary.each do |hash|
+        tag hash['Key'] => hash['Value']
+      end
+    end
+  end
+
+  # Unfortunately, Fog does not create fake instances after an auto scaling
+  # group is created.
+  def make_fake_instances
+    return unless Fog.mock?
+
+    asg_instances = []
+    all_instances = []
+    min_size.times do |n|
+      instance_id = Fog::AWS::Mock.instance_id
+      asg_instances << {
+        'AvailabilityZone'        => availability_zones,
+        'HealthStatus'            => 'Good',
+        'InstanceId'              => instance_id,
+        'LifecycleState'          => 'Pending',
+        'LaunchConfigurationName' => launch_configuration.aws_identifier
+      }
+
+      all_instances << {
+        'amiLaunchIndex'      => n,
+        'architecture'        => 'i386',
+        'blockDeviceMapping'  => [],
+        'clientToken'         => 'FAKE_CLIENT_TOKEN',
+        'dnsName'             => 'not-a-real-hostname',
+        'ebsOptimized'        => false,
+        'hypervisor'          => 'xen',
+        'imageId'             => launch_configuration.ami,
+        'instanceId'          => instance_id,
+        'instanceState'       => { 'code' => 0, 'name' => 'not pending?' },
+        'instanceType'        => launch_configuration.instance_type,
+        'kernelId'            => launch_configuration.kernel_id || Fog::AWS::Mock.kernel_id,
+        'keyName'             => launch_configuration.key_name,
+        'launchTime'          => Time.now,
+        'monitoring'          => { 'state' => false },
+        'placement'           => { 'availabilityZone' => availability_zones,
+                                   'groupName'        => self.aws_identifier,
+                                   'tenancy'          => 'default' },
+        'privateDnsName'      => nil,
+        'productCodes'        => [],
+        'reason'              => nil,
+        'rootDeviceType'      => 'instance-store',
+        'virtualizationType'  => 'paravirtual',
+        'groupIds'            => [],
+        'groupSet'            => launch_configuration.security_groups,
+        'iamInstanceProfile'  => launch_configuration.iam_role,
+        'networkInterfaces'   => [],
+        'ownerId'             => nil,
+        'privateIpAddress'    => nil,
+        'reservationId'       => Fog::AWS::Mock.reservation_id,
+        'stateReason'         => {},
+        'ipAddress'           => Fog::AWS::Mock.ip_address,
+        'privateIpAddress'    => Fog::AWS::Mock.private_ip_address
+      }
+    end
+    Aerosol::AWS.auto_scaling.data[:auto_scaling_groups][aws_identifier]
+                             .merge!('Instances' => asg_instances)
+    all_instances.each do |instance|
+      Aerosol::AWS.compute.data[:instances][instance['instanceId']] = instance
+    end
+  end
+end

data/lib/aerosol/aws.rb

@@ -0,0 +1,62 @@
+require 'fog/aws'
+
+# This module holds the connections for all AWS services used by the gem.
+module Aerosol::AWS
+  extend self
+
+  def service(name, klass)
+    define_method name do
+      if val = instance_variable_get(:"@#{name}")
+        val
+      else
+        instance = klass.new(creds)
+        instance_variable_set(:"@#{name}", instance)
+      end
+    end
+    services << name
+  end
+
+  def services
+    @services ||= []
+  end
+
+  def env_attr(*names)
+    names.each do |name|
+      define_method name do
+        instance_variable_get(:"@#{name}") || ENV[name.to_s.upcase]
+      end
+
+      define_method :"#{name}=" do |val|
+        reset_cache!
+        instance_variable_set(:"@#{name}", val)
+      end
+
+      env_attrs << name
+    end
+  end
+
+  def env_attrs
+    @env_attrs ||= []
+  end
+
+  def creds
+    attrs = Hash[env_attrs.map { |attr| [attr, public_send(attr)] }].reject { |k, v| v.nil? }
+    if attrs.empty?
+      if ENV['FOG_CREDENTIAL']
+        attrs = {} # let Fog use the env var
+      else
+        attrs = { :use_iam_profile => true }
+      end
+    end
+    attrs
+  end
+
+  def reset_cache!
+    services.each { |service| instance_variable_set(:"@#{service}", nil) }
+  end
+
+  service :s3, Fog::Storage::AWS
+  service :compute, Fog::Compute::AWS
+  service :auto_scaling, Fog::AWS::AutoScaling
+  env_attr :aws_access_key_id, :aws_secret_access_key
+end

data/lib/aerosol/aws_model.rb

@@ -0,0 +1,93 @@
+module Aerosol::AWSModel
+  def self.included(base)
+    base.instance_eval do
+      include Dockly::Util::DSL
+      extend ClassMethods
+
+      attr_accessor :from_aws
+    end
+  end
+
+  def initialize(hash={}, &block)
+    self.from_aws = false
+    super
+  end
+
+  def namespaced_name
+    Aerosol.namespace ? "#{Aerosol.namespace}-#{name}" : name.to_s
+  end
+
+  def default_identifier
+    "#{namespaced_name}-#{Aerosol::Util.git_sha}"
+  end
+
+  def create
+    raise '#create! must be defined to use #create' unless respond_to?(:create!)
+    create! unless exists?
+  end
+
+  def destroy
+    raise '#destroy! must be defined to use #destroy' unless respond_to?(:destroy!)
+    destroy! if exists?
+  end
+
+  def exists?
+    primary_value = send(self.class.primary_key)
+    self.class.exists?(primary_value)
+  end
+
+  module ClassMethods
+    def primary_key(attr = nil)
+      @primary_key = attr unless attr.nil?
+      @primary_key
+    end
+
+    def aws_attribute(hash)
+      dsl_attribute(*hash.keys)
+      aws_attributes.merge!(hash)
+    end
+
+    def aws_class_attribute(name, klass)
+      unless klass.ancestors.include?(Aerosol::AWSModel) || (klass == self)
+        raise '.aws_class_attribute requires a Aerosol::AWSModel that is not the current class.'
+      end
+
+      dsl_class_attribute(name, klass)
+      aws_class_attributes.merge!({ name => klass })
+    end
+
+    def exists?(key)
+      all.map { |inst| inst.send(primary_key) }.include?(key)
+    end
+
+    def all
+      raise 'Please define .request_all to use .all' unless respond_to?(:request_all)
+      request_all.map { |hash| from_hash(hash) }
+    end
+
+    def from_hash(hash)
+      raise 'To use .from_hash, you must specify a primary_key' if primary_key.nil?
+      refs = Hash[aws_class_attributes.map do |name, klass|
+        [name, klass.instances.values.find do |inst|
+          inst.send(klass.primary_key) &&
+            (inst.send(klass.primary_key) == hash[klass.aws_attributes[klass.primary_key]])
+        end]
+      end].reject { |k, v| v.nil? }
+
+      instance = new!
+      instance.from_aws = true
+
+      aws_attributes.each { |k, v| instance.send(k, hash[v]) unless hash[v].nil? }
+      refs.each { |name, inst| instance.send(name, inst.name) }
+      instance
+    end
+
+    def aws_attributes
+      @aws_attributes ||= {}
+    end
+
+    def aws_class_attributes
+      @aws_class_attributes ||= {}
+    end
+  end
+end

data/lib/aerosol/cli.rb

@@ -0,0 +1,41 @@
+require 'rubygems'
+require 'aerosol'
+require 'clamp'
+
+class Aerosol::AbstractCommand < Clamp::Command
+  option ['-f', '--file'], 'FILE', 'aerosol file to read', :default => 'aerosol.rb', :attribute_name => :file
+
+  def execute
+    if File.exist?(file)
+      Aerosol.load_file = file
+    else
+      raise 'Could not find an aerosol file!'
+    end
+  end
+end
+
+class Aerosol::SshCommand < Aerosol::AbstractCommand
+  option ['-r', '--run'], :flag, 'run first ssh command', :attribute_name => :run_first
+  parameter 'DEPLOY', 'the deploy to list commands for', :attribute_name => :deploy_name
+
+  def execute
+    super
+    if deploy = Aerosol.deploy(deploy_name.to_sym)
+      ssh_commands = deploy.generate_ssh_commands
+      raise 'No instances to ssh too!' if ssh_commands.empty?
+
+      ssh_commands.each do |ssh_command|
+        puts ssh_command
+      end
+
+      if run_first?
+        system(ssh_commands.first)
+      end
+    end
+  end
+end
+
+class Aerosol::Cli < Aerosol::AbstractCommand
+  subcommand ['ssh', 's'], 'Print ssh commands for latest running instances', Aerosol::SshCommand
+end
+

data/lib/aerosol/connection.rb

@@ -0,0 +1,39 @@
+require 'net/ssh'
+require 'net/ssh/gateway'
+
+class Aerosol::Connection
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[aerosol connection]'
+  dsl_attribute :user, :host, :jump
+
+  def with_connection(overridden_host=nil, &block)
+    actual_host = overridden_host || host
+    unless actual_host.is_a?(String)
+      actual_host = (actual_host.public_hostname || actual_host.private_ip_address)
+    end
+
+    if jump
+      info "connecting to gateway #{jump[:user] || user}@#{jump[:host]}"
+      gateway = nil
+      Timeout.timeout(20) do
+        gateway = Net::SSH::Gateway.new(jump[:host], jump[:user] || user, :forward_agent => true)
+      end
+
+      begin
+        info "connecting to #{user}@#{actual_host} through gateway"
+        gateway.ssh(actual_host, user, &block)
+      ensure
+        info "shutting down gateway connection"
+        gateway.shutdown!
+      end
+    else
+      info "connecting to #{user}@#{actual_host}"
+      Net::SSH.start(actual_host, user, &block)
+    end
+  rescue Timeout::Error => ex
+    error "Timeout error #{ex.message}"
+    error ex.backtrace.join("\n")
+  end
+end

data/lib/aerosol/deploy.rb

@@ -0,0 +1,105 @@
+class Aerosol::Deploy
+  include Dockly::Util::DSL
+  include Dockly::Util::Logger::Mixin
+
+  logger_prefix '[aerosol deploy]'
+  dsl_attribute :stop_command, :db_config_path,
+                :instance_live_grace_period, :app_port,
+                :continue_if_stop_app_fails, :stop_app_retries,
+                :sleep_before_termination, :post_deploy_command,
+                :ssl, :log_files, :tail_logs
+
+  dsl_class_attribute :ssh, Aerosol::Connection
+  dsl_class_attribute :migration_ssh, Aerosol::Connection
+  dsl_class_attribute :local_ssh, Aerosol::Connection
+  dsl_class_attribute :auto_scaling, Aerosol::AutoScaling
+
+  default_value :db_config_path, 'config/database.yml'
+  default_value :instance_live_grace_period, 30 * 60 # 30 Minutes
+  default_value :continue_if_stop_app_fails, false
+  default_value :stop_app_retries, 2
+  default_value :sleep_before_termination, 20
+  default_value :ssl, false
+  default_value :tail_logs, false
+  default_value :log_files, ['/var/log/syslog']
+
+  def live_check(arg = nil)
+    case
+    when arg.nil?
+      @live_check
+    when arg.start_with?('/')
+      @live_check = arg
+    else
+      @live_check = "/#{arg}"
+    end
+    @live_check
+  end
+
+  def is_alive?(&block)
+    @is_alive = block unless block.nil?
+    @is_alive
+  end
+
+  def live_check_url
+    [ssl ? 'https' : 'http', '://localhost:', app_port, live_check].join
+  end
+
+  def do_not_migrate!
+    self.instance_variable_set(:@db_config_path, nil)
+  end
+
+  def migration(opts = {})
+    self.db_config_path(opts[:db_config_path])
+  end
+
+  def migrate?
+    !!db_config_path
+  end
+
+  def local_ssh_ref
+    local_ssh || ssh
+  end
+
+  def run_post_deploy
+    return if post_deploy_command.nil?
+    info "running post deploy: #{post_deploy_command}"
+    if system(post_deploy_command)
+      info "post deploy ran successfully"
+      true
+    else
+      raise "post deploy failed"
+    end
+  end
+
+  def generate_ssh_command(instance)
+    ssh_command = "ssh -o 'UserKnownHostsFile=/dev/null' -o 'StrictHostKeyChecking=no' "
+    unless local_ssh_ref.nil?
+      unless local_ssh_ref.jump.nil? || local_ssh_ref.jump.empty?
+        ssh_command << "-o 'ProxyCommand=ssh -W %h:%p "
+        ssh_command << "#{local_ssh_ref.jump[:user]}@" if local_ssh_ref.jump[:user]
+        ssh_command << "#{local_ssh_ref.jump[:host]}' "
+      end
+      ssh_command << "#{local_ssh_ref.user}@" unless local_ssh_ref.user.nil?
+    end
+    ssh_command << "#{instance.public_hostname || instance.private_ip_address}"
+  end
+
+  def generate_ssh_commands
+    group = Aerosol::AutoScaling.latest_for_tag('Deploy', auto_scaling.namespaced_name)
+    raise "Could not find any auto scaling groups for this deploy (#{name})." if group.nil?
+
+    ssh_commands = []
+
+    with_prefix("[#{name}]") do |logger|
+      logger.info "found group: #{group.aws_identifier}"
+      instances = group.all_instances
+      raise "Could not find any instances for auto scaling group #{group.namespaced_name}" if instances.empty?
+      instances.each do |instance|
+        logger.info "printing ssh command for #{instance.public_hostname  || instance.private_ip_address}"
+        ssh_commands << generate_ssh_command(instance)
+      end
+    end
+
+    return ssh_commands
+  end
+end

data/lib/aerosol/env.rb

@@ -0,0 +1,6 @@
+# An environment is a set of deploys.
+class Aerosol::Env
+  include Dockly::Util::DSL
+
+  dsl_class_attribute :deploy, Aerosol::Deploy, type: Array
+end

data/lib/aerosol/instance.rb

@@ -0,0 +1,55 @@
+class Aerosol::Instance
+  include Aerosol::AWSModel
+
+  aws_attribute :availability_zone => 'AvailabilityZone',
+                :health_status     => 'HealthStatus',
+                :id                => 'InstanceId',
+                :lifecycle_state   => 'LifecycleState'
+  aws_class_attribute :launch_configuration, Aerosol::LaunchConfiguration
+  primary_key :id
+
+  def live?
+    describe_again
+    instance_state_name == 'running'
+  end
+
+  def instance_state_name
+    description['instanceState']['name']
+  end
+
+  def public_hostname
+    description['dnsName']
+  end
+
+  def private_ip_address
+    description['privateIpAddress']
+  end
+
+  def ami
+    description['imageId']
+  end
+
+  def description
+    @description ||= describe!
+  end
+
+  def self.request_all
+    Aerosol::AWS.auto_scaling
+                .describe_auto_scaling_instances
+                .body
+                .[]('DescribeAutoScalingInstancesResult')
+                .[]('AutoScalingInstances')
+  end
+
+private
+  def describe!
+    ensure_present! :id
+    result = Aerosol::AWS.compute.describe_instances('instance-id' => id).body
+    result['reservationSet'].first['instancesSet'].first rescue nil
+  end
+
+  def describe_again
+    @description = nil
+    description
+  end
+end