maws 0.8.0

data/lib/maws/connection.rb

@@ -0,0 +1,121 @@
+require 'right_aws'
+require 'maws/mash'
+require 'maws/description'
+
+class Connection
+  attr_reader :ec2, :rds, :elb
+
+  def initialize(config)
+    @config = config
+
+    @access_key_id = @config.aws_key.key_id
+    @secret_key = @config.aws_key.secret_key
+
+    @params = {:region => @config.region, :logger => $right_aws_logger}
+  end
+
+  def connect(services)
+    # The right_aws gem parses the EC2_URL environment variable if it is set. The EC2 CLI tools also use that variable
+    # but expect the hostname to be region-specific (e.g., us-east-1.ec2.amazonaws.com) instead of generic
+    # (e.g., ec2.amazonaws.com). To avoid conflicts, unset the variable here and use the right_aws default value.
+    ENV["EC2_URL"] = nil
+
+    # always connect to ec2
+    @ec2 = RightAws::Ec2.new(@access_key_id, @secret_key, @params.dup)
+
+    if services.include?(:rds)
+      @rds = RightAws::RdsInterface.new(@access_key_id, @secret_key, @params.dup)
+    end
+
+    if services.include?(:elb)
+      @elb = RightAws::ElbInterface.new(@access_key_id, @secret_key, @params.dup)
+    end
+  end
+
+  def available_zones
+    ec2.describe_availability_zones.
+        find_all{ |zone_description| zone_description[:zone_state] == "available"}.
+        map { |zone_description| zone_description[:zone_name][/\w$/]}
+  end
+
+  def image_id_for_image_name(image_name)
+    return if image_name.nil? || image_name.empty?
+    images = @ec2.describe_images(:filters => { 'tag:Name' => image_name})
+    if images.empty?
+      error "No AMI with name '#{image_name}'"
+    elsif images.count > 1
+      error "Ambigous AMI name: '#{image_name}'. Several AMIs match it #{images.collect{|i| i[:aws_id]}.join(', ')}"
+    else
+      images.first[:aws_id]
+    end
+  end
+
+  def descriptions(services = nil)
+    services ||= :all
+    descriptions = []
+
+    descriptions += ec2_descriptions if services == :all or services.include?(:ec2)
+    descriptions += rds_descriptions if services == :all or services.include?(:rds)
+    descriptions += elb_descriptions if services == :all or services.include?(:elb)
+
+    descriptions
+  end
+
+  def ec2_descriptions
+    # convert aws description to Description
+    descriptions =  descriptions = ec2.describe_instances.map {|description|
+      description[:service] = :ec2
+      Description.create(description)
+    }
+
+    # filter out terminated when same name exists as a living one and terminated
+    by_name = descriptions.group_by { |d| d.name }
+
+    # if there is one than more for the same name: delete terminated descriptions, take the last one (trust AWS sorting)
+    by_name.map {|name, descriptions|
+      if descriptions.count > 1
+        descriptions.delete_if {|d| d.status == "terminated"}
+        descriptions.replace([descriptions.last]).compact!
+      end
+    }
+
+    filter_current_profile_prefix(by_name.values.flatten)
+  end
+
+  def rds_descriptions
+    return [] unless rds
+
+    descriptions = rds.describe_db_instances.map { |description|
+      description[:service] = :rds
+      Description.create(description)
+    }
+
+    filter_current_profile_prefix(descriptions)
+  end
+
+  def elb_descriptions
+    return [] unless elb
+
+    descriptions = elb.describe_load_balancers.map { |description|
+      description[:service] = :elb
+      Description.create(description)
+    }
+
+    filter_current_profile_prefix(descriptions)
+  end
+
+  def ebs_descriptions
+    descriptions = ec2.describe_volumes.map { |description|
+      description[:service] = :ebs
+      Description.create(description)
+    }
+
+    filter_current_profile_prefix(descriptions)
+  end
+
+  private
+
+  def filter_current_profile_prefix(descriptions)
+    descriptions.delete_if {|d| d.profile != @config.profile.name || d.prefix != @config.prefix}
+  end
+end

data/lib/maws/core_ext/object.rb

@@ -0,0 +1,5 @@
+class Object
+  def blank?
+    respond_to?(:empty?) ? empty? : !self
+  end
+end

data/lib/maws/description/ebs.rb

@@ -0,0 +1,40 @@
+class Description::EBS < Description
+  def initialize(description)
+    description[:service] ||= :ebs
+    @description = description
+  end
+
+  def name
+    (tags && tags["Name"]) || aws_id
+  end
+
+  def status
+    aws_status
+  end
+
+  def region_zone
+    description[:zone]
+  end
+end
+
+# example EBS volume descriptions
+#  [{:snapshot_id=>"snap-a536ecc0",
+#    :service => :ebs, # this is set by maws
+#    :aws_id=>"vol-85fe67e8",
+#    :aws_status=>"available",
+#    :aws_created_at=>"2011-12-15T18:47:26.000Z",
+#    :zone=>"us-east-1c",
+#    :tags=>{},
+#    :aws_size=>100},
+#   {:aws_device=>"/dev/sda",
+#    :snapshot_id=>"snap-35446157",
+#    :aws_id=>"vol-6ce63401",
+#    :aws_status=>"in-use",
+#    :aws_created_at=>"2011-11-19T22:45:03.000Z",
+#    :aws_attachment_status=>"attached",
+#    :zone=>"us-east-1c",
+#    :tags=>{"Name"=>"foo-e2e-control-1"},
+#    :aws_size=>6,
+#    :aws_attached_at=>"2011-11-19T22:45:24.000Z",
+#    :aws_instance_id=>"i-fd126f9e",
+#    :delete_on_termination=>true},

data/lib/maws/description/ec2.rb

@@ -0,0 +1,72 @@
+class Description::EC2 < Description
+  def initialize(description)
+    description[:service] ||= :ec2
+    @description = description
+  end
+
+  def name
+    (tags && tags["Name"]) || aws_instance_id
+  end
+
+  def aws_id
+    aws_instance_id
+  end
+
+  def status
+    aws_state
+  end
+
+  def region_zone
+    aws_availability_zone
+  end
+
+  def ebs_volumes
+    block_device_mappings || []
+  end
+
+  def ebs_volume_ids
+    ebs_volumes.map {|v| v[:ebs_volume_id]}
+  end
+
+  def attached_ebs_volume_ids
+    ebs_volumes.find_all{|v| v[:ebs_status] == 'attached'}.map {|v| v[:ebs_volume_id]}
+  end
+end
+
+
+# example ec2 description
+# {:private_ip_address=>"10.240.7.99",
+#      :service => :ec2, # this is set by maws
+#      :aws_image_id=>"ami-c2a3f5d4",
+#      :ip_address=>"174.129.134.109",
+#      :dns_name=>"ec2-174-129-134-109.compute-1.amazonaws.com",
+#      :aws_instance_type=>"m1.small",
+#      :aws_owner=>"826693181925",
+#      :root_device_name=>"/dev/sda1",
+#      :instance_class=>"elastic",
+#      :aws_state=>"running",
+#      :private_dns_name=>"domU-12-31-39-04-00-95.compute-1.internal",
+#      :aws_reason=>"",
+#      :aws_launch_time=>"2009-11-18T14:03:25.000Z",
+#      :aws_reservation_id=>"r-54d38542",
+#      :aws_state_code=>16,
+#      :ami_launch_index=>"0",
+#      :aws_availability_zone=>"us-east-1a",
+#      :aws_groups=>["default"],
+#      :monitoring_state=>"disabled",
+#      :aws_product_codes=>[],
+#      :tags => {"Name" => "foo-test-web-01"}
+#      :ssh_key_name=>"",
+#      :block_device_mappings=>
+#       [{:ebs_status=>"attached",
+#         :ebs_delete_on_termination=>true,
+#         :ebs_attach_time=>"2009-11-18T14:03:34.000Z",
+#         :device_name=>"/dev/sda1",
+#         :ebs_volume_id=>"vol-e600f98f"},
+#        {:ebs_status=>"attached",
+#         :ebs_delete_on_termination=>true,
+#         :ebs_attach_time=>"2009-11-18T14:03:34.000Z",
+#         :device_name=>"/dev/sdk",
+#         :ebs_volume_id=>"vol-f900f990"}],
+#      :aws_instance_id=>"i-8ce84ae4"}
+

data/lib/maws/description/elb.rb

@@ -0,0 +1,52 @@
+class Description::ELB < Description
+  def initialize(description)
+    description[:service] ||= :elb
+    @description = description
+  end
+
+  def region_zone
+    # take from AZs it supports, ELBs always have at least one AZ
+    availability_zones.first
+  end
+
+  def physical_zone
+    nil
+  end
+
+  def name
+    description[:load_balancer_name]
+  end
+
+  def aws_id
+    description[:load_balancer_name]
+  end
+
+  def status
+    'available' if description[:load_balancer_name]
+  end
+
+  def enabled_zones
+    availability_zones.map {|z| z[-1,1]}
+  end
+end
+
+# example elb description
+# {:availability_zones=>["us-east-1c"],
+#  :service => :elb, # this is set by maws
+#  :dns_name=>"foo-lb-110916812.us-east-1.elb.amazonaws.com",
+#  :created_time=>"2011-07-06T23:50:06.040Z",
+#  :health_check=>
+#   {:timeout=>5,
+#    :target=>"HTTP:80/",
+#    :interval=>30,
+#    :healthy_threshold=>6,
+#    :unhealthy_threshold=>2},
+#  :instances=>["i-0941bf68", "i-a182d6c0"],
+#  :load_balancer_name=>"foo-lb",
+#  :app_cookie_stickiness_policies=>[],
+#  :lb_cookie_stickiness_policies=>[],
+#  :listeners=>
+#   [{:instance_port=>"80",
+#     :protocol=>"HTTP",
+#     :policy_names=>[],
+#     :load_balancer_port=>"80"}]}

data/lib/maws/description/rds.rb

@@ -0,0 +1,47 @@
+class Description::RDS < Description
+  def initialize(description)
+    description[:service] ||= :rds
+    @description = description
+  end
+
+  def name
+    aws_id
+  end
+
+  def region_zone
+    availability_zone
+  end
+
+  def logical_zone
+    # parse from name if not multi_az, otherwise nil
+    super unless multi_az
+  end
+
+end
+
+# example rds description
+# {:aws_id=>"foo-test-masterdb-1",
+#  :service => :rds, # this is set by maws
+#  :endpoint_port=>3306,
+#  :status=>"available",
+#  :multi_az=>true,
+#  :db_parameter_group=>{:status=>"in-sync", :name=>"default.mysql5.1"},
+#  :latest_restorable_time=>"2011-09-12T20:50:58.853Z",
+#  :master_username=>"root",
+#  :license_model=>"general-public-license",
+#  :engine=>"mysql",
+#  :pending_modified_values=>{},
+#  :db_security_groups=>[{:status=>"active", :name=>"default"}],
+#  :engine_version=>"5.1.57",
+#  :read_replica_db_instance_identifiers=>[],
+#  :availability_zone=>"us-east-1a",
+#  :backup_retention_period=>1,
+#  :create_time=>"2011-09-12T20:47:08.521Z",
+#  :auto_minor_version_upgrade=>true,
+#  :preferred_backup_window=>"07:30-08:00",
+#  :allocated_storage=>6,
+#  :instance_class=>"db.m1.small",
+#  :preferred_maintenance_window=>"mon:10:00-mon:10:30",
+#  :db_name=>"foo",
+#  :endpoint_address=>
+#   "foo-test-masterdb-1.ck6iyjop7iqg.us-east-1.rds.amazonaws.com"}

data/lib/maws/description.rb

@@ -0,0 +1,78 @@
+class Description
+  attr_reader :description
+
+  def self.create(description)
+    service = description[:service]
+    klass = Description.const_get("#{service.to_s.upcase}")
+    klass.new(description)
+  end
+
+  def [](key)
+    description[key]
+  end
+
+  def method_missing(method_name, *args, &block)
+    description[method_name.to_sym]
+  end
+
+  def initialize(description)
+    raise "use #{self.class}::create instead"
+  end
+
+  # logical zone (what the zone is according to maws categorization)
+  def zone
+    # us-east-1a => a
+    logical_zone
+  end
+
+  # zone where the instance really lives
+  # this has meaning for RDS with multi-zone = true and for EC2 with region scope
+  def physical_zone
+    region_zone[-1,1]
+  end
+
+  def region
+    # us-east-1a => us-east-1
+    region_zone[0...-1]
+  end
+
+  def name_re
+    /^(.+\.)?(.+)-(.+)-(\d+)(\w)?$/
+  end
+
+  def prefix
+    # old.foo-test-app-1z => old
+    md = name.match(name_re)
+    md[1].to_s[0...-1] if md
+  end
+
+  def profile
+    # foo-test-app-1z => foo-test
+    md = name.match(name_re)
+    md[2] if md
+  end
+
+  def role
+    md = name.match(name_re)
+    md[3] if md
+  end
+
+  def index
+    md = name.match(name_re)
+    md[4].to_i if md
+  end
+
+  def logical_zone
+    md = name.match(name_re)
+    md[5] if md
+  end
+
+  def create_instance(maws, config)
+    instance = Instance.create(maws, config, prefix, zone, role, index, {:service => service, :name => name, :region => region })
+    instance.description = self
+    instance
+  end
+end
+
+# load all description files
+Dir.glob(File.dirname(__FILE__) + '/description/*.rb') {|file| require file}

data/lib/maws/instance/ebs.rb

@@ -0,0 +1,45 @@
+require 'maws/instance'
+
+# for EC2 instances aws_id is a random id
+# name is a value of 'Name' tag
+class Instance::EBS < Instance
+
+  def destroy
+    connection.ec2.delete_volume(aws_id)
+    info "destroying EBS volume #{name}:#{device} (#{aws_id})"
+  end
+
+  def attached?
+    aws_attachment_status == 'attached'
+  end
+
+  def attached_to_instance_id
+    return "" unless attached_instance_name
+    device[:aws_instance_id]
+  end
+
+  def display_fields
+    if attached?
+      super + [:device, :aws_id, :attachment_status, :aws_instance_id]
+    else
+      super + [:aws_id]
+    end
+  end
+
+  def service
+    :ebs
+  end
+
+  def physical_zone
+    super || @config.available_zones.first
+  end
+
+  def device
+    description.aws_device
+  end
+
+  def attachment_status
+    description.aws_attachment_status
+  end
+end
+

data/lib/maws/instance/ec2.rb

@@ -0,0 +1,144 @@
+require 'maws/instance'
+require 'maws/ssh'
+
+# for EC2 instances aws_id is a random id
+# name is a value of 'Name' tag
+class Instance::EC2 < Instance
+  def create
+    return if alive?
+    info "creating EC2 #{name}..."
+    image_id = config(:image_id) || connection.image_id_for_image_name(config(:image_name))
+    if image_id.nil?
+      info "no AMI id found with name '#{config(:image_name)}'"
+      return
+    end
+
+    results = connection.ec2.launch_instances(image_id,
+      :availability_zone => region_physical_zone,
+      :key_name => config(:keypair),
+      :min_count => 1,
+      :max_count => 1,
+      :group_names => security_groups,
+      :user_data => config(:user_data),
+      :monitoring_enabled => config(:monitoring_enabled),
+      :instance_type => config(:instance_type))
+
+
+    self.description = Description::EC2.new(results.first)
+  end
+
+  def set_prefix(prefix)
+    @prefix = prefix
+    old_name = @name
+    @name = self.class.name_for(@config, @prefix, @zone, @role, @index)
+
+    info "renaming #{old_name} to #{@name}"
+    create_tags
+  end
+
+  def create_tags
+    tag_instance_name
+    tag_volumes_names
+
+    info "...done (#{name} or #{aws_id} is ready)"
+  end
+
+  def tag_instance_name
+    retries_left = 20
+    loop do
+      begin
+        connection.ec2.create_tags(aws_id, {'Name' => name})
+        info "tagged EC2 instance #{aws_id} as #{name}"
+        return
+      rescue RightAws::AwsError => error
+        if error.message =~ /^InvalidInstanceID.NotFound/
+          info "TAGGING FAILED. RETRYING..."
+        else
+          raise error
+        end
+      end
+
+      retries_left -= 1
+      retries_left > 0 ? sleep(1) : break
+    end
+    error "Couldn't not tag #{aws_id} with name #{name}. It might not exist on AWS"
+  end
+
+  def tag_volumes_names
+    # wait a while before volume_ids are made available on the aws ec2 description
+    retries_left = 30
+    loop do
+      if volume_ids.count > 0
+        # tag and return
+        volume_ids.each {|vid|
+          connection.ec2.create_tags(vid, {'Name' => name})
+          info "tagged EBS volume #{vid} as #{name}"
+        }
+        return
+      end
+
+      # resync all descriptions (volume ids should appear if missing)
+      @maws.resync_instances
+
+      retries_left -= 1
+      retries_left > 0 ? sleep(1) : break
+    end
+    error "No volumes found for #{name} (#{aws_id})"
+  end
+
+  def destroy
+    return unless alive?
+    connection.ec2.terminate_instances(aws_id)
+    info "destroying EC2 #{name} (#{aws_id})"
+  end
+
+  def stop
+    return unless alive?
+    if status == 'running'
+      connection.ec2.stop_instances(aws_id)
+      info "stopping EC2 #{name} (#{aws_id})"
+    end
+  end
+
+  def start
+    return unless alive?
+    if status == 'stopped'
+      connection.ec2.start_instances(aws_id)
+      info "starting EC2 #{name} (#{aws_id})"
+    end
+  end
+
+  def ssh_available?
+    return false unless alive? && self.dns_name && !self.dns_name.empty?
+
+    begin
+      3.times { # retry on host unreachable errors
+        begin
+          Net::SSH.start(dns_name, "phoneyuser", {:auth_methods => ["publickey"], :timeout => 1, :keys_only => true })
+        rescue Errno::EHOSTUNREACH
+          sleep 2
+        end
+      }
+    rescue Net::SSH::AuthenticationFailed
+      return true
+    rescue Object
+      return false
+    end
+  end
+
+  def volume_ids
+    description.ebs_volume_ids
+  end
+
+  def attached_volumes
+    description.attached_ebs_volume_ids
+  end
+
+  def service
+    :ec2
+  end
+
+  def display_fields
+    super + [:dns_name, :aws_instance_id, :aws_image_id]
+  end
+end

data/lib/maws/instance/elb.rb

@@ -0,0 +1,92 @@
+require 'maws/instance'
+
+# for ELBs aws_id is the same as their names
+class Instance::ELB < Instance
+  def create
+     return if alive?
+     info "creating ELB #{name}..."
+
+     listeners = config(:listeners).dup || []
+
+     server = connection.elb.create_load_balancer(name, @connection.availability_zones, listeners)
+     connection.elb.configure_health_check(name, config(:health_check))
+
+     description = server ? {:load_balancer_name => name} : {}
+     sync_from_description(description)
+
+     info "...done (ELB #{name} is ready)\n\n"
+   end
+
+  def destroy
+    return unless alive?
+    connection.elb.delete_load_balancer(aws_id)
+    info "destroying ELB #{name}"
+  end
+
+  def add_instances(instances)
+    names = instances.map{|i| i.name}
+    info "adding instances to ELB #{aws_id}: #{names.join(', ')}"
+    connection.elb.register_instances_with_load_balancer(aws_id, instances.map{|i| i.aws_id})
+    info "...done"
+  end
+
+  def remove_instances(instances)
+    names = instances.map{|i| i.name}
+    info "removing instances to ELB #{aws_id}: #{names.join(', ')}"
+    connection.elb.deregister_instances_with_load_balancer(aws_id, instances.map{|i| i.aws_id})
+    info "...done"
+  end
+
+  def attached_instances
+    instance_ids = @aws_description[:instances]
+    @profile.defined_instances.select {|i| instance_ids.include?(i.aws_id)}
+  end
+
+  def enable_zones(zones)
+    full_zones = zones.map {|z| command_options.region + z}
+    info "enabling zones #{full_zones.join(', ')} for ELB #{aws_id}..."
+    connection.elb.enable_availability_zones_for_load_balancer(aws_id, full_zones)
+    info "...done"
+  end
+
+  def disable_zones(zones)
+    full_zones = zones.map {|z| command_options.region + z}
+    info "disabling zones #{full_zones.join(', ')} for ELB #{aws_id}"
+
+    if enabled_availability_zones.size <= 1
+      info "can't remove last remaining zone: #{enabled_availability_zones.first}"
+      return
+    end
+
+    connection.elb.disable_availability_zones_for_load_balancer(aws_id, full_zones)
+    info "...done"
+  end
+
+  def service
+    :elb
+  end
+
+  def physical_zone
+    nil
+  end
+
+  def enabled_availability_zones
+    description.availability_zones
+  end
+
+  def zones_list
+    (description.enabled_zones || []).join(', ')
+  end
+
+  def display_fields
+    [:name, :status, :zones_list, :first_listener_info]
+  end
+
+  def first_listener_info
+    return "" unless alive?
+    (description.listeners || [{}]).first.to_hash.collect do |key, val|
+      "#{key}=#{val}"
+    end.join("; ")
+  end
+
+end