maws 0.8.0

data/lib/maws/commands/set-security-groups.rb

@@ -0,0 +1,442 @@
+require 'maws/command'
+
+class SetSecurityGroups < Command
+  def description
+    "set-security-groups - create or update security groups from security rules configuration"
+  end
+
+  # override from command to ignore selection
+  def verify_options
+    if @config.command_line.region.blank?
+      Trollop::die "Region must be specified in command line options OR as 'default_region' in #{@config.config.paths.config}"
+    end
+  end
+
+  def run!
+    rules = @config.security_rules
+    if rules.empty?
+      info "No security rules to create security groups from"
+      return
+    end
+
+    @security_group_definitions = {}
+    rules.keys.map {|rules_set_name|
+      security_group_name = "#{@config.profile.name}-#{rules_set_name}"
+      service = if rules_set_name == 'ec2_default'
+        security_group_name = 'ec2_default'
+        :ec2
+      elsif rules_set_name == 'rds_default'
+        security_group_name = 'rds_default'
+        :rds
+      else
+        @config.combined[rules_set_name].service
+      end
+
+      @security_group_definitions[security_group_name] = {
+              :role => rules_set_name,
+              :rules => rules[rules_set_name],
+              :service => service.to_sym}
+    }
+
+    update_definitions_with_descriptions(@security_group_definitions)
+
+
+    @security_group_definitions.each { |name, definition|
+      create_security_group(name, definition) unless definition[:description]
+    }
+
+    update_definitions_with_descriptions(@security_group_definitions)
+    begin
+      update_definitions_rules_with_real_ids(@security_group_definitions)
+    rescue BadSecurityRule => err
+      info "Bad security rule: #{err.message}"
+      info "No AWS security rules will be updated."
+      return
+    end
+
+    @security_group_definitions.each { |name, definition|
+      info "#{definition[:service]} security group #{name}"
+
+      clear_out_security_group(name, definition)
+
+      # rds takes a while to propagate. this code handles errors with retries, but this sleep keeps the noise down
+      if definition[:service] == :rds
+        info "...waiting for revoke to take effect..."
+        sleep 20
+      end
+
+      set_security_group(name, definition)
+
+      info "done\n\n"
+    }
+  end
+
+  # sets :description, :group_id, :group_name and :owner_id for each definition
+  def update_definitions_with_descriptions(security_group_definitions)
+    ec2_sg_descriptions = connection.ec2.describe_security_groups
+    rds_sg_descriptions = connection.rds.describe_db_security_groups
+
+    # attach existing descriptions (if they exist) to sec group definition
+    security_group_definitions.each {|name, definition|
+      if definition[:service] == :ec2
+        definition[:description] = ec2_sg_descriptions.find {|description| description[:aws_group_name] == name }
+        if definition[:description]
+          definition[:group_id] = definition[:description][:group_id]
+          definition[:group_name] = definition[:description][:aws_group_name]
+          definition[:owner_id] = definition[:description][:aws_owner]
+        end
+      elsif definition[:service] == :rds
+        definition[:description] = rds_sg_descriptions.find {|description| description[:name] == name }
+        if definition[:description]
+          definition[:group_id] = definition[:description][:name]
+          definition[:group_name] = definition[:description][:name]
+          definition[:owner_id] = definition[:description][:owner_id]
+        end
+      end
+    }
+  end
+
+
+
+  def create_security_group(name, definition)
+    info "CREATING #{name}"
+
+    if definition[:service] == :ec2
+      connection.ec2.create_security_group(name, name) # description same as name
+    elsif definition[:service] == :rds
+      connection.rds.create_db_security_group(name, name)
+    end
+  end
+
+  def clear_out_security_group(name, definition)
+    info "    clearing out #{name}"
+    description =  definition[:description]
+    return unless description && !description.empty?
+
+
+    if definition[:service] == :ec2
+      clear_out_ec2_security_group(name, description)
+    elsif definition[:service] == :rds
+      clear_out_rds_security_group(name, description)
+    end
+  end
+
+  def set_security_group(name, definition)
+    info "    adding security rules to #{name}"
+    service = definition[:service]
+
+    definition[:rules].each { |rule|
+      rule.port ||= 0
+      rule.port_from ||= (rule.port)
+      rule.port_to ||= (rule.port)
+
+      protocol_description = service == :ec2 ? "#{rule.protocol}:#{rule.port_from}-#{rule.port_to}" : ""
+
+      if rule.group
+        info "        allow from #{rule.owner_id}/#{rule.group_id} (#{rule.group}) #{protocol_description}"
+        if service == :ec2
+          authorize_ec2_security_group_rule(definition[:group_id], rule.port_from, rule.port_to, rule.protocol,
+                                            :groups => {rule.owner_id => rule.group_id})
+        else
+          safe_authorize_rds_security_group_rule(definition[:group_id],
+                                            :ec2_security_group_owner => rule.owner_id,
+                                            :ec2_security_group_name => rule.group_name)
+        end
+      elsif rule.role
+        info "        allow from #{rule.owner_id}/#{rule.group_id} (role '#{rule.role}') #{protocol_description}"
+        if service == :ec2
+          authorize_ec2_security_group_rule(definition[:group_id], rule.port_from, rule.port_to, rule.protocol,
+                                            :groups => {rule.owner_id => rule.group_id})
+        else
+          safe_authorize_rds_security_group_rule(definition[:group_id],
+                                            :ec2_security_group_owner => rule.owner_id,
+                                            :ec2_security_group_name => rule.group_id)
+        end
+      elsif rule.cidr
+        info "        allow from #{rule.cidr.inspect} #{protocol_description}"
+        if service == :ec2
+          authorize_ec2_security_group_rule(definition[:group_id], rule.port_from, rule.port_to, rule.protocol,
+                                            :cidr_ips => [rule.cidr].flatten)
+        else
+          [rule.cidr].flatten.each { |cidr|
+            safe_authorize_rds_security_group_rule(definition[:group_id], :cidrip => cidr)
+          }
+        end
+      end
+    }
+  end
+
+  ### EC2 ###
+
+  def clear_out_ec2_security_group(name, description)
+    description[:aws_perms].each { |permission|
+      group_id = description[:group_id]
+
+      revoke_params = if permission[:group_id]
+        # this is a group rule
+        {:groups => {permission[:owner] => permission[:group_id]}}
+      else
+        {:cidr_ip => permission[:cidr_ips]}
+      end
+
+      revoke_params[:protocol] = permission[:protocol]
+      revoke_params[:from_port] = permission[:from_port]
+      revoke_params[:to_port] = permission[:to_port]
+
+      info "        revoking #{revoke_params.inspect}"
+      connection.ec2.modify_security_group(:revoke, :ingress,
+        group_id, revoke_params);
+    }
+  end
+
+  def authorize_ec2_security_group_rule(group_id, from_port, to_port, protocol, rule)
+    params = {:from_port  => from_port,
+              :to_port    => to_port,
+              :protocol   => protocol}.merge(rule)
+    connection.ec2.modify_security_group(:authorize, :ingress, group_id, params)
+  end
+
+
+
+  ### RDS ###
+
+  def clear_out_rds_security_group(name, description)
+    description[:ec2_security_groups].each {|group_permission|
+      info "        revoking from #{group_permission[:owner_id]}/#{group_permission[:name]}"
+      safe_revoke_rds_security_group(name,
+                                       :ec2_security_group_owner => group_permission[:owner_id],
+                                       :ec2_security_group_name  => group_permission[:name])
+    }
+
+    description[:ip_ranges].each {|ip_permission|
+      info "        revoking from #{ip_permission[:cidrip].inspect}"
+      safe_revoke_rds_security_group(name, :cidrip => ip_permission[:cidrip])
+    }
+  end
+
+
+  def update_definitions_rules_with_real_ids(definitions)
+    definitions.each { |name, definition|
+      definition[:rules].each { |rule|
+        next if rule.cidr
+
+        owner_id, group_id = if rule.role
+          resolve_owner_and_group_id_for_role_definition(rule.role)
+        elsif rule.group
+          resolve_owner_and_group_id_for_group_definition(rule.group)
+        end
+
+        unless owner_id && group_id
+          raise BadSecurityRule.new("Can't resolve security group id and owner for rule #{rule.inspect}}")
+        else
+          rule.owner_id = owner_id
+          rule.group_id = group_id
+        end
+      }
+    }
+
+  end
+
+
+  def resolve_owner_and_group_id_for_role_definition(role_name)
+    definition = @security_group_definitions.values.find {|definition| definition[:role] == role_name}
+    if definition
+      p [definition[:owner_id], definition[:group_id]]
+      [definition[:owner_id], definition[:group_id]]
+    else
+      security_group_name = "#{@config.profile.name}-#{role_name}"
+      raise BadSecurityRule.new("Can't find security group '#{security_group_name}' for role '#{role_name}'")
+    end
+  end
+
+  def resolve_owner_and_group_id_for_group_definition(group_definition)
+    # 'owner_id/group_name' => ['owner_id', 'group_name'] OR 'group_name' => ['group_name']
+    owner_id_and_group = group_definition.split('/')
+
+    group = owner_id_and_group.pop
+    owner_id = owner_id_and_group.pop
+
+    group_id = if group =~ /sg-/
+      group
+    else
+      # look up name
+      definition = @security_group_definitions.values.find {|definition| definition[:group_name] == group}
+
+      definition[:group_id] if definition
+    end
+
+    raise BadSecurityRule.new("Can't find group id (sg-...) for group name #{group}") unless group_id
+
+    # use this account for owner id
+    owner_id = @security_group_definitions.values.first[:owner_id] unless owner_id
+
+    [owner_id, group_id]
+  end
+
+
+  def safe_revoke_rds_security_group(name, params)
+    rds_logger = connection.rds.logger
+    connection.rds.logger = NullLogger.new
+
+    tries = 4
+
+    loop do
+      if tries <= 0
+        info "!!!!!! FAILED TO REVOKE: #{name} #{params.inspect} !!!!!!"
+        return
+      end
+
+      begin
+        connection.rds.revoke_db_security_group_ingress(name, params)
+        info "            (succesfully revoked)"
+        return
+      rescue Exception => e
+        if e.message =~ /AuthorizationNotFound/
+          info "            (not authorized. nothing to do here)"
+          return
+
+        elsif e.message =~ /InvalidDBSecurityGroupState: Cannot revoke an authorization that is in the revoking state/
+          info "            (already revoking - will be revoked shortly)"
+          sleep 10
+          tries -= 1
+
+        elsif e.message =~ /InvalidDBSecurityGroupState: Cannot revoke an authorization that is in the authorizing state/
+          info "            (not yet finished authorizing. waiting and retrying...)"
+          sleep 10
+          tries -= 1
+
+        else
+          sleep 1
+          tries -= 1
+        end
+      end
+    end
+  ensure
+    connection.rds.logger = rds_logger
+  end
+
+
+  def safe_authorize_rds_security_group_rule(name, params)
+    rds_logger = connection.rds.logger
+    connection.rds.logger = NullLogger.new
+
+    tries = 4
+
+    loop do
+      if tries <= 0
+        info "!!!!!! FAILED TO AUTHORIZE: #{name} #{params.inspect} !!!!!!"
+        return
+      end
+
+      begin
+        connection.rds.authorize_db_security_group_ingress(name, params)
+        info "            (succesfully authorized)"
+        return
+      rescue Exception => e
+        if e.message =~ /AuthorizationAlreadyExists/
+          info "            (authorization already exists. probably means it is still being revoked. retrying...)"
+          sleep 10
+          tries -= 1
+
+        elsif e.message =~ /InvalidDBSecurityGroupState: Cannot authorize an authorization that is in the revoking state/
+          info "            (currently revoking - will be revoked shortly. will retry authorizing then)"
+          sleep 10
+          tries -= 1
+
+        elsif e.message =~ /InvalidDBSecurityGroupState: Cannot authorize an authorization that is in the authorizing state/
+          info "            (already authorizing. waiting and retrying to confirm...)"
+          sleep 10
+          tries -= 1
+
+        else
+          sleep 1
+          tries -= 1
+        end
+      end
+    end
+  ensure
+    connection.rds.logger = rds_logger
+  end
+
+
+##### DESCRIPTIONS ######
+
+### EC2 ###
+
+# ec2.describe_security_groups #=>
+#       [{:aws_perms=>
+#         [{:owner=>"375390957666",
+#           :direction=>:ingress,
+#           :protocol=>"tcp",
+#           :group_id=>"sg-d7e9d9be",
+#           :from_port=>"22",
+#           :group_name=>"foo-e2e-queue",
+#           :to_port=>"22"},
+
+#          {:cidr_ips=>"208.240.243.170/32",
+#            :direction=>:ingress,
+#            :protocol=>"tcp",
+#            :from_port=>"22",
+#            :to_port=>"22"}],
+
+#          {:owner=>"375390957666",
+#           :direction=>:ingress,
+#           :protocol=>"icmp",
+#           :group_id=>"sg-9f5d5cf6",
+#           :from_port=>"15",
+#           :group_name=>"foo-e2e-service",
+#           :to_port=>"-1"}],
+
+#        :aws_owner=>"375390957666",
+#        :aws_description=>"something here",
+#        :group_id=>"sg-30fd1b58",
+#        :aws_group_name=>"test2"}]
+
+# connection.ec2.modify_security_group(:grant, :ingress,
+#     'sg-30fd1b58', {:protocol => 'tcp', :port => 22, :cidr_ip => '127.0.0.2/32'})
+#
+# connection.ec2.modify_security_group(:revoke, :ingress,
+#     'sg-30fd1b58', {:protocol => :tcp, :port => 22, :groups => {'375390957666' => 'sg-d7e9d9be'} });
+
+
+
+### RDS ###
+
+# rds.describe_db_security_groups #=>
+#   [{:owner_id=>"375390957666",
+#     :description=>"Default",
+#     :ec2_security_groups=>[],
+#     :ip_ranges=>[],
+#     :name=>"Default"},
+#    {:owner_id=>"375390957666",
+#     :description=>"kd",
+#     :ec2_security_groups=>[],
+#     :ip_ranges=>[],
+#     :name=>"kd2"},
+#    {:owner_id=>"375390957666",
+#     :description=>"kd",
+#     :ec2_security_groups=>
+#      [{:status=>"Authorized", :owner_id=>"375390957666", :name=>"default"},
+#       {:status=>"Authorized", :owner_id=>"375390957666", :name=>"default1"},
+#       {:status=>"Authorized", :owner_id=>"375390957666", :name=>"default"},
+#       {:status=>"Authorized", :owner_id=>"375390957666", :name=>"default"},
+#       {:status=>"Authorized", :owner_id=>"375390957666", :name=>"default1"},
+#       {:status=>"Authorized", :owner_id=>"375390957666", :name=>"default22"}],
+#     :ip_ranges=>
+#      [{:status=>"Authorized", :cidrip=>"127.0.0.1/8"},
+#       {:status=>"Authorized", :cidrip=>"128.0.0.1/8"},
+#       {:status=>"Authorized", :cidrip=>"129.0.0.1/8"},
+#       {:status=>"Authorized", :cidrip=>"130.0.0.1/8"},
+#       {:status=>"Authorized", :cidrip=>"131.0.0.1/8"}],
+#     :name=>"kd3"}]
+
+# rds.revoke_db_security_group_ingress('kd2', :ec2_security_group_owner => '375390957666',
+#                                            :ec2_security_group_name => 'default')
+#
+# rds.revoke_db_security_group_ingress('kd2', :cidrip=>"127.0.0.1/8")
+#
+# rds.authorize_db_security_group_ingress('kd3', :cidrip => '131.0.0.1/8')
+end
+
+class BadSecurityRule < Exception
+end

data/lib/maws/commands/start.rb

@@ -0,0 +1,11 @@
+require 'maws/command'
+
+class Start < Command
+  def description
+    "start - start specified EC2 instances that were stopped"
+  end
+
+  def run!
+    instances.specified.each {|i| i.start if i.respond_to?(:start)}
+  end
+end

data/lib/maws/commands/status.rb

@@ -0,0 +1,25 @@
+require 'maws/command'
+require 'terminal-table/import'
+
+class Status < Command
+  def description
+    "status - show information about specified instances"
+  end
+
+  def run!
+    instances.specified
+    if instances.specified.empty?
+      info "no instances specified"
+      return
+    end
+
+    roles_list = @config.available_roles
+
+    instances.specified.roles_in_order_of(roles_list).each { |role_name|
+      role_instances = instances.specified.with_role(role_name)
+      next if role_instances.empty?
+      InstanceDisplay.display_collection_for_role(role_name, role_instances)
+    }
+
+  end
+end

data/lib/maws/commands/stop.rb

@@ -0,0 +1,11 @@
+require 'maws/command'
+
+class Stop < Command
+  def description
+    "stop - stop specified EC2 instances that are running"
+  end
+
+  def run!
+    instances.specified.each {|i| i.stop if i.respond_to?(:stop)}
+  end
+end

data/lib/maws/commands/teardown.rb

@@ -0,0 +1,11 @@
+require 'maws/command'
+
+class Teardown < Command
+  def description
+    "teardown - destroys all instances that are not specified, but exist on AWS"
+  end
+
+  def run!
+    instances.aws.not_specified.alive.each {|i| i.destroy}
+  end
+end

data/lib/maws/commands/volumes-cleanup.rb

@@ -0,0 +1,22 @@
+require 'maws/command'
+require 'maws/volumes_command'
+require 'terminal-table/import'
+
+class VolumesCleanup < VolumesCommand
+  def description
+    "volumes-cleaned - delete unattached EBS volumes for specified roles"
+  end
+
+  def run!
+    super
+
+    unattached = instances.specified.ebs.matching(:attached? => false)
+
+    if unattached.empty?
+      info "no unattached volumes to clean up"
+      return
+    end
+
+    unattached.each {|i| i.destroy}
+  end
+end

data/lib/maws/commands/volumes-status.rb

@@ -0,0 +1,43 @@
+require 'maws/command'
+require 'maws/volumes_command'
+require 'terminal-table/import'
+
+class VolumesStatus < VolumesCommand
+  def description
+     "volumes-status - show brief status information for EBS volumes for specified roles"
+  end
+
+  def run!
+    super
+    attached = instances.specified.ebs.matching(:attached? => true)
+    unattached = instances.specified.ebs.matching(:attached? => false)
+
+    info "\n**** " + "ATTACHED EBS VOLUMES" + " *****************"
+    list_ebs_instances(attached)
+
+    info "\n\n**** " + "UNATTACHED EBS VOLUMES" + " *****************"
+    list_ebs_instances(unattached)
+  end
+
+  def list_ebs_instances(instances)
+    if instances.empty?
+      info "none available"
+      return
+    end
+
+    headers = instances.first.display.headers
+    table_rows = []
+    grouped_instances = instances.
+                            group_by{|i| i.name}.
+                            to_a.sort_by {|group| group[0]} # sort by name
+
+    grouped_instances.each_with_index do |(name, instances), i|
+      instances.each {|instance|
+        table_rows << instance.display.values
+      }
+
+    end
+
+    info table(headers, *table_rows)
+  end
+end

data/lib/maws/commands/wait.rb

@@ -0,0 +1,61 @@
+require 'maws/command'
+require 'maws/trollop'
+
+class Wait < Command
+  def description
+    "wait - do nothing until specified instances enter specified state, then quite and notify"
+  end
+
+  def add_specific_options(parser)
+    parser.opt :target_state, "State to wait for", :type => :string
+    parser.opt :wait, "Max wait (seconds)", :default => 60
+    parser.opt :count, "Minimum number of specified instances that must match the state", :default => 0
+    parser.opt :quiet, "Be quiet", :type => :flag, :default => false
+    parser.opt :growl, "Growl notify when done", :type => :flag, :default => false
+  end
+
+  def verify_options
+    super
+    state = @config.command_line.target_state
+    Trollop::die "Can't wait for blank state" if state.nil? || state.empty?
+  end
+
+  def run!
+    at_exit do
+      if @config.command_line.growl
+        system("growlnotify -m 'waiting for AWS state #{@config.command_line.target_state} done'")
+      end
+    end
+
+    state = @config.command_line.target_state
+    been_waiting = 0
+    wait_for_count = @config.command_line.count == 0 ? instances.specified.count : @config.command_line.count
+    wait_for_time = @config.command_line.wait
+
+    info "waiting #{wait_for_time} seconds or until #{wait_for_count} are #{state}..."
+
+    loop do
+      trap("INT") { info "...done (interrupted)"; return }
+      return if been_waiting >= wait_for_time
+      left_to_wait = wait_for_time - been_waiting
+      matching_count = instances.specified.with_approximate_status(state).count
+
+      if matching_count >= wait_for_count
+        info "...done (#{matching_count}/#{wait_for_count} are #{state})"
+        return
+      end
+
+      if @config.command_line.quiet
+        print "."
+        $stdout.flush
+      else
+        info "#{matching_count}/#{wait_for_count} are #{state} - wait #{left_to_wait} seconds or until #{wait_for_count}/#{wait_for_count} are #{state}..."
+      end
+
+
+      sleep 1
+      been_waiting += 1
+      @maws.resync_instances
+    end
+  end
+end