maws 0.8.0

data/bin/maws

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+
+# code_base_path = File.expand_path("../lib", File.dirname(__FILE__))
+# $LOAD_PATH << code_base_path
+
+p $LOAD_PATH
+require 'rubygems'
+require 'maws'
+
+Maws.load_and_run!

data/lib/maws/chunk_source.rb

@@ -0,0 +1,41 @@
+class ChunkSource
+  def initialize(config, instances)
+    @config = config
+    @instances = instances
+  end
+
+  # what is :first, :all, :chunk
+  # options are:
+  #   :chunk_size - how big the chunks are
+  #   :chunk_key  - unique name of the context for looping over all instances in chunks. every time calling this method with same key will return next chunk
+  #   :from       - nil to pick default scope for role_name, otherwise :zone or :region
+  def select(what, role_name, zone, options = {})
+    # :zone or :region
+    from_zone = (options[:from] || @config.combined[role_name].scope).to_sym == :zone
+
+    source_instances = from_zone ? @instances.with_zone(zone) : @instances
+    source_instances = source_instances.with_role(role_name)
+
+    what = :all if what == :chunk && options[:chunk_size].nil?
+
+    case what
+    when :all : source_instances
+    when :first : source_instances.first
+    when :chunk : pick_next_chunk(source_instances, options[:chunk_size], options[:chunk_key])
+    end
+  end
+
+  def pick_next_chunk(all, size = 1, key = "")
+    # this is the starting index of the current chunk
+    # this index doesn't wrap around, it needs to be wrapped by the lookup methiod
+    @chunk_indexes ||= {}
+    @chunk_indexes[key] ||= 0
+
+    index = @chunk_indexes[key] % all.count # real index
+    @chunk_indexes[key] += size # update for next time this is called
+
+    # so we don't have to do any math to wrap it
+    big_all = all * size
+    big_all[index,size]
+  end
+end

data/lib/maws/command.rb

@@ -0,0 +1,62 @@
+require 'maws/instance'
+
+class Command
+  attr_accessor :maws, :connection
+
+  def description
+    "command - does nothing"
+  end
+
+  def instances
+    maws.instances
+  end
+
+
+  def initialize(config)
+    @config = config
+    @maws = nil
+  end
+
+  def connection
+    @maws.connection
+  end
+
+  def run!
+    puts "generic command does nothing"
+  end
+
+
+  def add_generic_options(parser)
+    available_roles = @config.available_roles.join(', ')
+    parser.opt :selection, "Selection of instances
+         any of the following:
+             zones (a, b, c, etc.)
+             indexes (2, 1-10, 5-, -10, *) where * is all existing indexes and blank means all defined in profile
+             role names (available: #{available_roles})
+         in any order and separated by spaces", :type => :string, :default => nil
+    parser.opt :prefix, "Prefix", :short => '-p', :type => :string, :default => ""
+    parser.opt :region, "Region", :short => '-R', :type => :string, :default => @config.config.default_region
+  end
+
+  def add_specific_options(parser)
+  end
+
+  def process_options
+    @config.region = @config.command_line.region
+    @config.prefix = @config.command_line.prefix
+  end
+
+  def verify_options
+    if @config.command_line.region.blank?
+      Trollop::die "Region must be specified in command line options OR as 'default_region' in #{@config.config.paths.config}"
+    end
+
+    if @config.command_line.selection.nil?
+      Trollop::die "Must specify a selection of some instances"
+    end
+  end
+
+  def verify_configs
+  end
+
+end

data/lib/maws/command_loader.rb

@@ -0,0 +1,28 @@
+class CommandLoader
+  def initialize(config)
+    @config = config
+  end
+
+  def load
+    # assumes command file is known to exist already
+    @config.command_name = @config.command_line.command_name
+    command_path = @config.config.available_commands[@config.command_name]
+    class_constant_name = constantize_file_name(@config.command_name)
+
+    require command_path
+
+    begin
+      @config.command_class = Object.const_get(class_constant_name)
+    rescue NameError
+      error "Could not load class #{class_constant_name} from the file #{command_path}"
+      exit(1)
+    end
+  end
+
+  private
+
+  def constantize_file_name(dashed)
+    dashed.split('-').map {|w| w.capitalize}.join
+  end
+
+end

data/lib/maws/command_options_parser.rb

@@ -0,0 +1,37 @@
+require 'maws/trollop'
+
+class CommandOptionsParser
+  def initialize(config)
+    @config = config
+  end
+
+  def process_command_options(command)
+    banner_text = usage(@config.profile.name, @config.command_name)
+    banner_text << "\n#{command.description}\n\n"
+    banner_text << "options:\n"
+    command_opts = Trollop::options do
+      banner banner_text
+      command.add_generic_options(self)
+      command.add_specific_options(self)
+    end
+
+    @config.command_line.merge!(command_opts)
+  end
+
+  def usage(profile = nil, command = nil)
+    profile ||= 'profile'
+    command ||= 'command'
+    <<-EOS
+
+MAWS - Toolset for provisioning and managing AWS
+
+Usage:
+       maws.rb #{profile} #{command} [options]
+
+profiles: #{@config.config.available_profiles.keys.join(', ')}
+commands: #{@config.config.available_commands.keys.join(', ')}
+
+    EOS
+  end
+
+end

data/lib/maws/commands/configure.rb

@@ -0,0 +1,287 @@
+require 'maws/command'
+require 'maws/chunk_source'
+require 'erubis'
+require 'maws/ssh'
+require 'net/scp'
+require 'fileutils'
+
+
+
+class Configure < Command
+  attr_reader :options
+  def description
+    "configure - create and upload templated configurations and run commands on specified servers with SSH"
+  end
+
+  def add_specific_options(parser)
+    parser.opt :dump, "Dump config files before uploading them", :type => :flag, :default => false
+    parser.opt :command, "Command to run remotely (either name or a string)", :type => :string, :default => ""
+    parser.opt :login_name, "The SSH login name", :short => '-l', :type => :string, :default => "root"
+    parser.opt :hostname, "The SSH hostname", :short => '-h', :type => :string, :default => nil
+    parser.opt :identity_file, "The SSH identity file", :short => '-i', :type => :string
+    parser.opt :copy_to_local, "Copy template output to local folders", :short => '-L', :type => :flag, :default => false
+  end
+
+  def run!
+    @chunk_source = ChunkSource.new(@config, instances)
+    @template_output_dir = @config.config.paths.template_output
+
+    @ssh_actions = {}
+    @options = @config.command_line
+
+    configurable_instances = instances.specified.find_all do |instance|
+      (instance.status == 'running' || instance.configure_without_running) &&
+      (instance.configurations || !options.command.empty?)
+    end
+
+    prepare_ssh_actions(configurable_instances)
+    execute_ssh_actions(configurable_instances)
+  end
+
+  def prepare_ssh_actions(instances)
+    instances.each do |instance|
+      @ssh_actions[instance] = []
+      build_ssh_actions_for_instance(instance)
+    end
+  end
+
+  def execute_ssh_actions(instances)
+    # create threads that execute ssh commands
+    threads = []
+    ssh_connections = {}
+    instances.each do |instance|
+      ssh_actions = @ssh_actions[instance]
+      next if ssh_actions.empty?
+
+      threads << Thread.new do
+        Thread.current[:title] = "#{instance.name} (#{instance.dns_name})"
+        Thread.current[:ensured_output] = []
+
+        ssh = ssh_connect_to(instance)
+        ssh_connections[instance] = ssh
+
+        ssh_actions.each {|action| action.call(ssh)}
+
+        ssh_disconnect(ssh, instance)
+        ssh_connections[instance] = nil
+      end
+    end
+
+    begin
+      threads.each do |t|
+        Kernel.sleep 0.03
+        t.join
+      end
+    ensure
+      ssh_connections.each do |instance, ssh|
+        ssh_disconnect(ssh, instance) if ssh
+      end
+      print_ensured_output(threads)
+    end
+  end
+
+  def build_ssh_actions_for_instance(instance)
+    if instance.configurations && instance.configurations.collect{|c| c.name}.include?(options.command)
+      # command specified as name of a config
+      configuration = instance.configurations.find{|c| c.name == options.command}
+      execute_configuration(instance, configuration)
+    elsif options.command && !options.command.empty?
+      queue_remote_command(instance, nil, options.command)
+    end
+  end
+
+  def ssh_connect_to(instance)
+    host = options.hostname || instance.dns_name
+    user = options.login_name
+
+    identity_file = options.identity_file || File.join(KEYPAIRS_PATH, "#{instance.keypair}.pem")
+    raise ArgumentError.new("Missing identity file: #{identity_file}") if !File.exists?(identity_file)
+
+    info "connecting to '#{user}@#{host}'..."
+
+    3.times { # retry on host unreachable errors
+      begin
+        return Net::SSH.start(host, user, { :keys => [identity_file], :verbose => :warn, :auth_methods => ["publickey"], :keys_only => true })
+      rescue Errno::EHOSTUNREACH
+        sleep 2
+      end
+    }
+  end
+
+  def ssh_disconnect(ssh, instance)
+    host = options.hostname || instance.dns_name
+    user = options.login_name
+
+    info "...done (disconnected from '#{user}@#{host}')"
+    ssh.close
+  end
+
+  def execute_configuration(instance, configuration)
+    if configuration.template
+      generate_and_queue_upload_template(instance, configuration)
+    elsif configuration.command
+      queue_remote_command(instance, configuration.name, configuration.command)
+    elsif configuration.command_set
+      configuration.command_set.to_a.each do |command_name|
+        specified_configuration = instance.configurations.find{|c| c.name == command_name}
+        execute_configuration(instance, specified_configuration) if specified_configuration
+      end
+    end
+  end
+
+  def queue_remote_command(instance, name, command)
+    queue_ssh_action(instance) {|ssh| do_remote_command(ssh, instance, name, command)}
+  end
+
+  def do_remote_command(ssh, instance, name, command)
+    if name
+      ensure_output :info, "   executing #{name} command: " + command
+    else
+      ensure_output :info, "   executing #{command}"
+    end
+
+    result = ssh.exec!(command)
+    ensure_output :info, result if result
+  end
+
+  def generate_and_queue_upload_template(instance, configuration)
+    locations = [configuration.location].flatten
+    locations.each do |location|
+      # prepare params for config file interpolation
+      resolved_params = {}
+      configuration.template_params ||= {}
+
+      configuration.template_params.each do |param_name, param_config|
+        resolved_params[param_name] = resolve_template_param(instance, configuration.template, param_name, param_config)
+      end
+      resolved_params['instance'] = instance
+      resolved_params['instances'] = instances
+      resolved_params['settings'] = @profile.settings
+
+      # generate config file
+      template_path = File.join(TEMPLATES_PATH, configuration.template + ".erb")
+      template = File.read(template_path)
+      generated_config =  Erubis::Eruby.new(template).result(resolved_params)
+
+      Dir.mkdir(TEMPLATE_OUTPUT_DIR) if !File.directory?(TEMPLATE_OUTPUT_DIR)
+      config_output_path = File.join(TEMPLATE_OUTPUT_DIR, "#{instance.name}--#{instance.aws_id}." + configuration.template)
+      File.open(config_output_path, "w") {|f| f.write(generated_config)}
+      info "generated  '#{config_output_path}'"
+
+      if options.copy_to_local
+        info "copying to local path: #{location}"
+        FileUtils.cp(config_output_path, location)
+      end
+
+      queue_ssh_action(instance) do |ssh|
+        ensure_output :info, "configuring #{configuration.template} for #{instance.name}"
+
+        if options.dump
+          ensure_output :info, "\n\n------- BEGIN #{config_output_path} -------"
+          ensure_output :info, generated_config
+          ensure_output :info, "-------- END #{config_output_path} --------\n\n"
+        end
+
+        if configuration.copy_as_user
+          remote_tmp_path = File.join("/tmp/", "#{instance.name}--#{instance.aws_id}." + configuration.template)
+          ssh.scp.upload!(config_output_path, remote_tmp_path)
+          cp_command = "sudo su - #{configuration.copy_as_user} -c 'cp -f #{remote_tmp_path} #{location}' && sudo rm #{remote_tmp_path}"
+          do_remote_command(ssh, instance, 'copy_as_user', cp_command)
+        else
+          ssh.scp.upload!(config_output_path, location)
+        end
+
+        if configuration.owner
+          chown_command = "sudo su - -c 'chown -R #{configuration.owner} #{location}'"
+          do_remote_command(ssh, instance, 'permissions', chown_command)
+        end
+
+        if configuration.permissions
+          chmod_command = "sudo su - -c 'chmod -R #{configuration.permissions} #{location}'"
+          do_remote_command(ssh, instance, 'permissions', chmod_command)
+        end
+
+        timestamp = ssh.exec!("sudo stat -c %y #{location}")
+        ensure_output :info, "            new timestamp for #{location}: " + timestamp
+      end
+    end
+  end
+
+  def resolve_template_param(instance, template_name, param_name, param_config)
+    if param_config == 'self'
+      return instance
+
+    elsif param_config == 'config'
+      return @config
+
+    elsif param_config.is_a? String
+      return param_config
+
+    elsif param_config.select_one.is_a? String
+      context = "#{instance.role_name}-#{template_name}-#{param_name}"
+      @chunk_source.select(:chunk, param_config.select_one, instance.zone,
+                              :chunk_size => 1, :chunk_key => context).first
+
+    elsif param_config.select_many.is_a? String
+      @chunk_source.select(:all, param_config.select_many, instance.zone)
+
+    elsif param_config.select_one
+      context = "#{instance.role_name}-#{template_name}-#{param_name}"
+      from = param_config.select_one.from # nil means default
+      @chunk_source.select(:chunk, param_config.select_one.role, instance.zone,
+                                    :chunk_size => 1, :chunk_key => context, :from => from).first
+
+    elsif param_config.select_many
+      context = "#{instance.role_name}-#{template_name}-#{param_name}"
+      from = param_config.select_many.from # nil means default
+      @chunk_source.select(:chunk, param_config.select_many.role, instance.zone,
+                                  :chunk_size => param_config.select_many.chunk_size, :chunk_key => context, :from => from)
+
+    else
+      param_config
+    end
+  end
+
+  def verify_configs
+    @config.roles.each do |name, config|
+      verify_config(name, config, "role definition")
+    end
+
+    @config.profile.each do |name, config|
+      verify_config(name, config, "profile role")
+    end
+  end
+
+  private
+
+  def verify_config(name, config, scope = '')
+    return unless config.respond_to? :configurations
+
+    Trollop::die "empty or non-array configurations for #{scope} '#{name}'" unless config.configurations.respond_to? :each
+
+    config.configurations.each do |configuration|
+      Trollop::die "nil configuration for #{scope} '#{name}'" unless configuration
+      Trollop::die "nameless configuration [#{configuration.to_hash.inspect}] for #{scope} '#{name}'" if configuration.name.to_s.empty?
+    end
+  end
+
+  def queue_ssh_action(instance, &block)
+    @ssh_actions[instance] << block
+  end
+
+  def print_ensured_output(threads)
+    info "\n"
+    threads.each do |t|
+      InstanceDisplay.pretty_describe_heading(t[:title])
+      t[:ensured_output].each do |method, message|
+        send(method, message)
+      end
+
+      InstanceDisplay.pretty_describe_footer
+    end
+  end
+
+  def ensure_output method, message
+    Thread.current[:ensured_output] << [method.to_sym, message]
+  end
+end

data/lib/maws/commands/console.rb

@@ -0,0 +1,38 @@
+require 'maws/command'
+
+class Console < Command
+  def description
+    "console - interactive console for direct access to MAWS internal objects"
+  end
+
+  def run!
+    IRB.start_session(binding)
+  end
+end
+
+
+
+require 'irb'
+
+module IRB # :nodoc:
+  def self.start_session(binding)
+    unless @__initialized
+      args = ARGV
+      ARGV.replace(ARGV.dup)
+      IRB.setup(nil)
+      ARGV.replace(args)
+      @__initialized = true
+    end
+
+    workspace = WorkSpace.new(binding)
+
+    irb = Irb.new(workspace)
+
+    @CONF[:IRB_RC].call(irb.context) if @CONF[:IRB_RC]
+    @CONF[:MAIN_CONTEXT] = irb.context
+
+    catch(:IRB_EXIT) do
+      irb.eval_input
+    end
+  end
+end

data/lib/maws/commands/create.rb

@@ -0,0 +1,25 @@
+require 'maws/command'
+
+class Create < Command
+  def description
+    "create - creates all specified instances if they don't exist"
+  end
+
+  def run!
+    instances_to_create = instances.specified.not_alive
+    if instances_to_create.empty?
+      info "nothing to create"
+      return
+    end
+
+    instances_to_create.each {|i| i.create}
+
+    # create tags
+    instances_to_be_tagged =  instances_to_create.alive.with_service(:ec2)
+    unless instances_to_be_tagged.empty?
+      sleep 2
+      # connection.silent = true
+      instances_to_be_tagged.each {|i| i.create_tags}
+    end
+  end
+end

data/lib/maws/commands/describe.rb

@@ -0,0 +1,15 @@
+require 'maws/command'
+require 'terminal-table/import'
+
+class Describe < Command
+  def description
+    "describe - prints all available AWS information for specified instances"
+  end
+
+  def run!
+    instances.specified.each do |i|
+      i.display.pretty_details
+    end
+  end
+
+end

data/lib/maws/commands/destroy.rb

@@ -0,0 +1,11 @@
+require 'maws/command'
+
+class Destroy < Command
+  def description
+    "destroy - permenantly delete specified instances"
+  end
+
+  def run!
+    instances.specified.alive.each {|i| i.destroy}
+  end
+end

data/lib/maws/commands/elb-add.rb

@@ -0,0 +1,17 @@
+require 'maws/command'
+require 'maws/elb_command'
+require 'maws/trollop'
+
+class ElbAdd < ElbCommand
+  def description
+    "elb-add - adds all specified EC2 instances to all specified ELBs"
+  end
+
+  def run!
+    ec2s = instances.specified.with_service(:ec2)
+    elbs = instances.specified.with_service(:elb)
+
+    elbs.each {|elb| elb.add_instances(ec2s)}
+  end
+end
+

data/lib/maws/commands/elb-describe.rb

@@ -0,0 +1,23 @@
+require 'maws/command'
+require 'maws/elb_command'
+require 'maws/trollop'
+
+class ElbDescribe < ElbCommand
+  def description
+    "elb-describe - prints detailed information about all specified ELBs"
+  end
+
+  def run!
+    elbs = instances.specified.with_service(:elb)
+
+    elbs.each do |elb|
+      title = elb.name.to_s.upcase
+      instances = elb.attached_instances
+      text =  "ENABLED ZONES: #{elb.enabled_availability_zones.join(', ')}\n" +
+      "ATTACHED INSTANCES:\n\n" +
+      instances.collect {|i| "#{i.name} (#{i.aws_id})"}.join("\n")
+
+      InstanceDisplay.pretty_describe(title, text)
+    end
+  end
+end

data/lib/maws/commands/elb-disable-zones.rb

@@ -0,0 +1,17 @@
+require 'maws/command'
+require 'maws/elb_command'
+require 'maws/trollop'
+
+class ElbDisableZones < ElbCommand
+  def description
+    "elb-disable-zones - remove entire zones from specified ELBs (ELBs will not allow removing the last zone)"
+  end
+
+  def add_specific_options(parser)
+    parser.opt :zones, "Zones to disable", :short => '-z', :type => :strings
+  end
+
+  def run!
+    instances.specified.with_service(:elb).each {|elb| elb.disable_zones(options.zones)}
+  end
+end

data/lib/maws/commands/elb-enable-zones.rb

@@ -0,0 +1,18 @@
+require 'maws/command'
+require 'maws/elb_command'
+require 'maws/trollop'
+
+class ElbEnableZones < ElbCommand
+  def description
+    "elb-enable-zones - add entire zones to specified ELBs (zones must exist)"
+  end
+
+
+  def add_specific_options(parser)
+    parser.opt :zones, "Zones to enable", :short => '-z', :type => :strings
+  end
+
+  def run!
+    instances.specified.with_service(:elb).each {|elb| elb.enable_zones(options.zones)}
+  end
+end

data/lib/maws/commands/elb-remove.rb

@@ -0,0 +1,16 @@
+require 'maws/command'
+require 'maws/elb_command'
+require 'maws/trollop'
+
+class ElbRemove < ElbCommand
+  def description
+    "elb-remove - remove specified EC2 instances from specified ELBs"
+  end
+
+  def run!
+    ec2s = instances.specified.with_service(:ec2)
+    elbs = instances.specified.with_service(:elb)
+
+    elbs.each {|elb| elb.remove_instances(ec2s)}
+  end
+end

data/lib/maws/commands/set-prefix.rb

@@ -0,0 +1,24 @@
+require 'maws/command'
+
+class SetPrefix < Command
+  def description
+    "set-prefix - sets prefix for specified EC2 instances"
+  end
+
+  def add_specific_options(parser)
+    parser.opt :prefix_to_set, "Prefix to set", :short => "-P", :type => :string, :default => ""
+  end
+
+  def verify_options
+    super
+    prefix = @config.command_line.prefix_to_set
+
+    Trollop::die "Can't set the prefix to be the same as the prefix currently in scope: '#{@config.prefix}'" if prefix == @config.prefix
+  end
+
+  def run!
+    prefix = @config.command_line.prefix_to_set
+
+    instances.specified.alive.each {|i| i.set_prefix(prefix) if i.respond_to?(:set_prefix)}
+  end
+end