cluster 0.5.33

data/lib/cluster/cli.rb

@@ -0,0 +1,206 @@
+require 'cluster'
+
+class Cluster
+  class Cli
+    def initialize(arguments = nil)
+      arguments ||= ARGV
+
+      unless arguments.length >= 1
+        puts "#{Cluster::NAME} usage: #{Cluster::NAME} [infa args] command [command args]"
+        exit 1
+      end
+
+      if arguments.include?('--version')
+        puts "#{Cluster::NAME} version #{Cluster::Version::STRING}"
+        exit 0
+      end
+
+      infra = []
+      command = nil
+      params = []
+      for arg in arguments
+        if command
+          params << arg
+        elsif arg =~ /^-/
+          infra << arg
+        else
+          command = arg
+        end
+      end
+
+      if arg = infra.detect {|a| a =~ /^(-c|--credentials=)(.+)/ }
+        file = $2
+        file = (file[0, 1] == File::SEPARATOR) ? file : File.join(ENV['PWD'], file)
+        
+        Cluster::Configuration[:credentials_file] = file
+        infra.delete arg
+      else
+        Cluster.set_credentials_file
+      end
+
+      logger_file = if arg = infra.detect {|a| a =~ /^--logger=(.+)$/ }
+                      infra.delete arg
+                      $1
+                    elsif Cluster::Configuration[:credentials_file]
+                      File.join(File.dirname(Cluster::Configuration[:credentials_file]), 'cluster.log')
+                    else
+                      '/tmp/cluster.log'
+                    end
+
+      unless command
+        puts "#{Cluster::NAME} usage: #{Cluster::NAME} [infa args] command [command args]"
+        exit 1
+      end
+
+      if command.downcase == 'gemurl'
+        # We don't need to do any infrastructure for this one...and 
+        # we may not have it yet...
+        puts Cluster::LOCATION
+        exit 0
+      elsif command.downcase == 'imageurl'
+        # We don't need to do any infrastructure for this one...and 
+        # we may not have it yet...
+        puts Cluster::IMAGES
+        exit 0
+      end
+
+      file = File.open(logger_file, File::WRONLY | File::APPEND | File::CREAT)
+
+      $stderr = file
+      Cluster::Configuration['logger'] = Logger.new(file, 5, 512000)
+
+      @sub = Infrastructure.connect(infra)
+
+      @named_output = (!params.empty? and params[0] == '-d' and params.shift)
+      @ip_output = (!params.empty? and params[0] == '-i' and params.shift)
+      @id_output = (!params.empty? and params[0] == '-I' and params.shift)
+
+      @cluster = Cluster.new @sub
+      if respond_to? command
+        self.send command, *params
+      elsif @cluster.respond_to? command
+        puts @cluster.send(command, *params)
+      else
+        STDERR.puts "#{Cluster::NAME} does not understand #{command}."
+      end
+    end
+
+    def machines(*params)
+      res = @cluster.machines(*params)
+      print_instances(res)
+    end
+
+    def services(*params)
+      res = @cluster.services(*params)
+      print_instances(res)
+    end
+    alias :service :services
+
+    def labeled(*params)
+      print_instances @cluster.labeled(params.first)
+    end
+
+    def list(*params)
+      print_instances(@sub.instances) if params.empty?
+
+      print_instances @sub.instances.select {|i| params.any? {|p| i.identified_by? p } }
+    end
+
+    def print_instances(instances)
+      return if instances.empty?
+
+      for instance in instances
+        if @named_output
+          puts instance.dns
+        elsif @ip_output
+          puts instance.ip
+        elsif @id_output
+          puts instance.id
+        else
+          puts instance.to_s(:long)
+        end
+      end
+    end
+
+    def alter_instances(ids)
+      altered = @sub.instances.map {|ins|
+        if ids.any? {|p| ins.identified_by? p}
+          yield ins
+          ins
+        else
+          nil
+        end
+      }.compact
+
+      @sub.alter_instances! altered
+      puts "Instances altered :"
+      for ins in altered 
+        puts ins.to_s(:long)
+      end
+    end
+
+    def enable(*params)
+      service_list = params.shift
+
+      if !service_list or params.empty?
+        STDERR.puts "Usage: #{Cluster::NAME} [infra] enable service[,services] instance [instances]"
+        exit 2
+      end
+
+      servs = service_list.split(',')
+      alter_instances(params) {|ins| ins.enable servs }
+    end
+
+    def disable(*params)
+      service_list = params.shift
+
+      if !service_list or params.empty?
+        STDERR.puts "Usage: #{Cluster::NAME} [infra] disable service[,services] instance [instances]"
+        exit 2
+      end
+
+      servs = service_list.split(',')
+      alter_instances(params) {|ins| ins.disable servs }
+    end
+
+    def label(name, id)
+      alter_instances([id]) {|i| i.friendly_name = name.downcase}
+    end
+
+    def unlabel(id)
+      alter_instances([id]) {|i| i.friendly_name = ''}
+    end
+
+    def set_state(state, *ids)
+      alter_instances(ids) {|i| i.set_state state }
+    end
+
+    def stop(*ids)
+      alter_instances(ids) {|i| i.stop! }
+    end
+
+    def release(*params)
+      if params.empty?
+        puts "Need an environment to work on for releases."
+        exit 1
+      end
+
+      tag_output = (!params.empty? and params.delete('-t'))
+      rel = @cluster.release(*params)
+      if !rel
+        puts "No release found for environment #{params.first}"
+        exit 1
+      elsif tag_output
+        puts rel.tag
+      else
+        puts "#{rel.environment} was released at #{rel.created_at} with '#{rel.tag}'"
+      end
+    end
+
+    def security(*args)
+      @cluster.security(*args).each {|k, v|
+        puts "#{k}: #{v.join(' ')}"
+      }
+    end
+  end
+end

data/lib/cluster/configuration.rb

@@ -0,0 +1,52 @@
+require 'logger'
+
+class Cluster
+  module Logging
+    # Logging should be set up by configuration early, but if it isnt
+    # then we will just write to STDERR
+    @@logger = nil
+
+    def logger
+      @@logger ||= Cluster::Configuration['logger']
+    end
+  end
+
+  class Configuration
+    include Cluster::Logging
+
+    @@system_config = {}
+    @@credentials = nil
+
+    class << self
+      def credentials?
+        self[:credentials_file]
+      end
+
+      def credentials
+        @@credentials ||= credentials? && File.open(self[:credentials_file]) {|f| YAML::load(f) }
+      end
+
+      def []=(key, value)
+        @@system_config[key.to_s] = value
+      end
+
+      def [](key)
+        @@system_config[key.to_s]
+      end
+
+      def method_missing(meth, *params)
+        field = meth.to_s
+        if options.respond_to? field
+          options.send(field)
+        else
+          super
+        end
+      end
+
+      def options(params = nil)
+        @@system_config ||= self.new(params)
+      end
+    end
+  end
+
+end

data/lib/cluster/infrastructure.rb

@@ -0,0 +1,160 @@
+%w(ostruct optparse).each {|l| require l }
+
+class Infrastructure
+  include Cluster::Logging
+
+  @@subsystem = nil
+  @@cluster_name = 'cluster'
+  @@in_cluster = false
+  @@machine_sizes = %w(minimum basic average power super)
+
+  def initialize(arguments = nil)
+    @arguments = arguments
+  end
+
+  def configure
+    @options = OpenStruct.new
+    @options.current_instance_id = nil
+    @options.original_services = []
+
+    if credentials?
+      cluster = credentials['cluster']
+      if cluster
+        @options.current_instance_id = cluster['id']
+        @options.original_services = cluster['services'].split(',')
+      end
+    end
+
+    @@subsystem ||= self
+  end
+
+  def instances
+    raise NotImplementedError
+  end
+
+  def current_instance
+    return nil unless @options.current_instance_id
+
+    instances.detect {|ins| ins.id.eql?(@options.current_instance_id) }
+  end
+
+
+  def credentials?
+    return true if @credentials
+
+    if Cluster::Configuration.credentials?
+      @credentials = Cluster::Configuration.credentials
+    end
+  end
+
+  def credentials
+    @credentials
+  end
+
+  def in_cluster?
+    raise NotImplementedError
+  end
+
+  def names(*roles)
+    raise NotImplementedError
+  end
+
+  def release_class
+    raise NotImplementedError
+  end
+
+  def machines(filter_groups)
+    if filter_groups.empty?
+      self.instances
+    else
+      self.instances.select {|i| i.groups.any? {|g| filter_groups.include? g }}
+    end
+  end
+
+  def services(filter_services)
+    filter_services = Array(filter_services) unless filter_services.is_a? Array
+
+    if filter_services.empty?
+      puts "No services provided to return a list for #{filter_services.join(' ')}"
+      exit 1
+    else
+      self.instances.select {|i|
+        i.services.any? {|s| filter_services.include?(s) and !i.disabled_services.include?(s)}
+      }
+    end
+  end
+
+  def to_credentials
+    opts = @options.marshal_dump.keys.inject({}) {|m, k| 
+      if k.eql? :role
+        m
+      else
+        m.merge k.to_s => @options.send(k)
+      end
+    }
+    {'cluster_name' => self.class.cluster_name,
+      self.class.to_s.downcase => opts }
+  end
+
+  @@dns = nil
+  class << self
+    def sizes
+      @@machine_sizes
+    end
+    alias :machine_sizes :sizes
+
+    def current
+      @@subsystem
+    end
+
+    def cluster_name
+      @@cluster_name
+    end
+
+    def connect(arguments = nil)
+      name = nil
+      # removes the infrastructure argument, if given, and processes it
+      new_args = arguments.map {|arg|
+        case arg
+        when /^-i([^=]+)$/, /^--infr[^=]+=(.+)$/
+          name = $1
+          nil
+        when /^-n([^=]+)$/, /^--name[^=]+=(.+)$/
+          @@cluster_name = $1
+        else
+          arg
+        end
+      }.compact
+
+      name ||= 'amazon'
+
+      begin
+        require File.join('cluster', 'infrastructures', name)
+      rescue LoadError => err
+        STDERR.puts "Subsystem of #{name} either is not included or not available.\n\t#{err.message}\n#{err.backtrace.join("\n\t")}"
+        exit 1
+      end
+
+      begin
+        sub = Object.const_get name.capitalize
+      rescue NameError
+        STDERR.puts "Subsystem of #{name.capitalize} cannot be initialized."
+        exit 1
+      end
+
+      sub.new new_args
+    end
+
+    def dns
+      return @@dns unless @@dns.nil?
+      ns = ''
+      File.open('/etc/resolv.conf').each do |line| ns = $1 if line =~ /nameserver\s+([\d\.]+)/; end
+
+      @@dns = Resolv::DNS.new(:nameserver => ns)
+    end
+
+    def in_cluster?
+      @@in_cluster
+    end
+  end
+end

data/lib/cluster/infrastructures/amazon.rb

@@ -0,0 +1,568 @@
+%w(uuidtools yaml right_aws resolv sdb/active_sdb cluster/infrastructures/amazon_release cluster/infrastructures/amazon_instance).each {|l| require l}
+
+class Amazon < Infrastructure
+
+  def configure
+    super
+    @@in_cluster = in_cluster?
+    @@instances = nil
+
+    if credentials? and credentials.include? 'amazon'
+      creds = credentials['amazon']
+      @options.key = creds['key']
+      @options.secret = creds['secret']
+      @options.owner = creds['owner']
+      @options.cluster_bucket = creds['cluster_bucket']
+      @options.bucket_key = creds['bucket_key']
+      @options.cluster_domain = creds['cluster_domain']
+      @options.zone = creds['zone']
+      @options.volumes = creds['volumes']
+    end
+
+    @options.key ||= ENV['AMAZON_ACCESS_KEY_ID'] 
+    @options.secret ||= ENV['AMAZON_SECRET_ACCESS_KEY'] 
+    @options.owner ||= ENV['AMAZON_OWNER_ID'] 
+    @options.cluster_bucket ||= ENV['CLUSTER_BUCKET']
+    @options.bucket_key ||= 'cluster_credentials.yml'
+    @options.zone ||= ENV['AMAZON_ZONE']
+    @options.cluster_domain ||= ENV['CLUSTER_DOMAIN'] || self.class.cluster_name
+    @options.volumes ||= {}
+
+    @options.role = (ENV['CLUSTER_ROLE'] or ENV['RAILS_ENV'] or 'production')
+
+    @options.cluster_image_key = 'cluster_images.yml'
+    @options.spot_instances = false
+    @options.price = false
+
+    OptionParser.new {|o|
+      o.banner = "Amazon Infrastructure Options"
+
+      o.on('-k', '--key VAL', "Amazon Access Key ID") do |v|
+        @options.key = v
+      end
+
+      o.on('-s', '--secret VAL', 'Amazon Access Secret') do |v|
+        @options.secret = v
+      end
+
+      o.on('-o', '--owenr VAL', 'Amazon User Code') do |v|
+        @options.owner = v
+      end
+
+      o.on('-d', '--domain VAL', 'Amazon Domain to use') do |v|
+        @options.cluster_domain = v
+      end
+
+      o.on('-b', '--bucket VAL', 'Cluster Bucket') do |v|
+        @options.cluster_bucket = v
+      end
+
+      o.on('-f', '--bucket-credentials-file VAL', 'Cluster credentials file location on the bucket.') do |v|
+        @options.bucket_key = v
+      end
+
+      o.on('--source-bucket VAL', 'Bucket that has the host data.') do |v|
+      end
+
+      o.on('-r', '--role VAL', 'Role in which to operate.') do |v|
+        @options.role = v
+      end
+
+      o.on('-z', '--zone VAL', "Availability Zone") do |v|
+        @options.zone = v
+      end
+
+      o.on('--spot', "Use Spot Instances") do |v|
+        @options.spot_instances = true
+      end
+
+      o.on('--price=VAL', 'Maximum price for the spot instance') do |v|
+        @options.price = v
+      end
+
+    }.parse(@arguments)
+
+    unless @options.key and @options.secret
+      $stderr.puts "Amazon Infrastructure cannot communicate without secret and key."
+      exit 2
+    end
+
+    unless @options.cluster_domain
+      $stderr.puts "Amazon Infrastructure needs to know what domain to connect to."
+      exit 2
+    end
+  end
+
+  def instances
+    @@instances ||= load_instances
+  end
+
+  def cost(sizes)
+    unless @options.spot_instances
+      puts "Cost only works for spot instances currently (ie. supply infrastructure argument of --spot"
+      exit 3
+    end
+    args = {:start_time => (Time.now - (7 * 24 * 60 * 60)),
+          :end_time => Time.now,
+          :product_description => "Linux/UNIX"}
+    sizes.map {|s|
+      t = [size_to_instance_type(s)]
+      prices = ecc.describe_spot_price_history(args.merge(:instance_types => t))
+      p = prices.reduce(0) {|c,p| c + p[:spot_price] } / prices.length
+      [s, "%0.3f" % p]
+    }
+  end
+
+  def create_file_store(name)
+    sss.bucket(name, true)
+  end
+
+  def create_data_store(name)
+    sdb.create_domain(name)
+  end
+
+  def load_instances
+    terminated = []
+    iss = ecc.describe_instances.map {|ins|
+      if %w(terminated shutting-down).include? ins[:aws_state]
+        terminated << ins[:aws_instance_id]
+        nil
+      else
+        AmazonInstance.new(ins)
+      end
+    }.compact
+
+    begin
+      res = sdb.select "select * from #{domain} where entry='machine'"
+    rescue RightAws::AwsError
+      unless domains.include? domain
+        sdb.create_domain domain 
+        retry
+      end
+    end
+    sdbs = self.class.from_sdb_results res
+
+    sdbs.each do |sd|
+      aid = sd[:aws_id]
+      iid = sd['ec2_id']
+      if !iid
+        started = sd['start_time_sorted'] && Time.parse(sd['start_time_sorted'])
+        diff = started && (Time.now - started)
+        if diff and diff < (12 * 3600)
+          ins = AmazonInstance.new
+          ins.set_sdb_attributes sd
+          iss.push ins
+        else
+          $stderr.puts "Cannot find machine #{aid} -- old entry being removed."
+          sdb.delete_attributes domain, aid
+        end
+      else
+        if ins = iss.detect {|i| i.id.eql? iid }
+          ins.set_sdb_attributes sd
+        elsif terminated.include? iid
+          $stderr.puts "Removing terminated entry #{iid}"
+          sdb.delete_attributes domain, aid
+        else
+          $stderr.puts "Orphaned cluster record of #{aid}.  (Just started?) [#{sd.inspect}]"
+        end
+      end
+    end
+
+    iss.each do |ins|
+      if ins.no_sdb?
+        puts "Cannot find cluster registration for #{ins.ec2_id} -- creating."
+        sdb.put_attributes domain, ins.aws_id, ins.attributes, :replace
+      end
+    end
+
+    iss
+  end
+
+  def alter_instances!(*iss)
+    list = iss.empty? ? instances : iss
+
+    for ins in list.flatten
+      yield ins if block_given?
+      attrs = ins.attributes
+      remove = attrs.keys.select {|k| attrs[k].empty? and attrs.delete(k) }
+      unless remove.empty?
+        sdb.delete_attributes domain, ins.aws_id, remove
+      end
+      sdb.put_attributes domain, ins.aws_id, ins.attributes, :replace
+    end
+  end
+
+  def in_cluster?
+    return @cluster_check if @cluster_check
+
+    check = false
+    begin
+      Timeout::timeout(1) do
+        begin
+          s = TCPSocket.new('169.254.169.254', 80)
+          s.close
+          check = true
+        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+          # NOP
+        end
+      end
+    rescue Timeout::Error
+    end
+
+    @@in_cluster = check
+  end
+
+  def size_to_instance_type(size)
+    unless self.class.sizes.include? size.downcase
+      puts "#{Cluster::NAME} does not have a machine size of #{size}\n\tAvailable Sizes: (#{self.class.sizes.join(', ')})"
+      exit 2
+    end
+    AmazonInstance.size_to_type(size)
+  end
+
+  def new_instance(size, services)
+    args = { :services => services, :size => size }
+    if @options.spot_instances
+      unless @options.price
+        puts "Amazon Spot instances need a '--price=??' argument"
+        exit 3
+      end
+      args.merge! :spot_price => @options.price
+    end
+=begin
+       type = size_to_type size
+    groups = services_to_groups services
+    image = type_to_image type
+    puts "type #{type} Groups #{groups.inspect} K #{key} I #{image} UD #{user}"
+=end
+    ins = AmazonInstance.create args
+    puts ins.inspect
+    ins
+  end
+
+  def save_monitor(io, key)
+    if bucket.put key, io
+      old = sdb.select "select * from #{domain} where entry = 'monitor'"
+      monitor = unless old[:items].empty?
+                  self.class.from_sdb_results(old).first
+                else
+                  { 'aws_id' => UUIDTools::UUID.timestamp_create.to_s,
+                    'entry' => 'monitor' }
+                end
+      monitor.merge! 'updated_at' => Time.now.xmlschema(3),
+        'key' => key,
+        'bucket' => @cluster_bucket
+      sdb.put_attributes domain, monitor['aws_id'], self.class.to_sdb_attributes(monitor), :replace
+      monitor
+    end
+  end
+
+  def fetch_monitor
+    res = sdb.select "select * from #{domain} where entry = 'monitor'"
+    return nil if res[:items].empty?
+    monitor = self.class.from_sdb_results(res).first
+    bucket.get(monitor['key'])
+  end
+
+  def current_instance
+    unless in_cluster?
+      puts "Are we in the cluster?"
+      exit 3
+    end
+
+    require 'open-uri'
+    ec2_id = open('http://169.254.169.254/latest/meta-data/instance-id') {|f|  f.read }
+    ins = self.instances.detect {|i| i.ec2_id.eql? ec2_id }
+
+    unless ins
+      puts "#{Cluster::NAME} cannot determine the current instance." 
+      exit 2
+    end
+
+    if @options.current_instance_id
+      aws = self.instances.detect {|i| i.aws_id.eql? @options.current_instance_id }
+      if aws and aws.aws_id != ins.aws_id
+        ins.services = (ins.services + aws.services).uniq
+        ins.disabled_services = (ins.disabled_services + aws.disabled_services).uniq
+        ins.spot_price = aws.spot_price unless ins.spot_price
+        ins.friendly_name = aws.friendly_name unless ins.friendly_name
+        sdb.delete_attributes domain, aws.aws_id
+        sdb.put_attributes domain, ins.aws_id, ins.attributes, :replace
+      end
+    end
+
+    ins
+  end
+
+  def release_class
+    AmazonRelease
+  end
+
+  # The methods below here are typically only used internally, but
+  # could also be called by anything that would like to access
+  # the Amazon tools directly.
+
+  def key
+    @options.key
+  end
+
+  def secret
+    @options.secret
+  end
+
+  def owner
+    @options.owner
+  end
+
+  def domain
+    @options.cluster_domain
+  end
+
+  def sdb(params = {})
+    return @sdb if @sdb
+    @sdb = RightAws::SdbInterface.new(key, secret, connection_params(params))
+    unless @sdb
+      puts "Amazon cannot connect to SimpleDB"
+      exit 3
+    end
+    @sdb
+  end
+
+  def connect_to_active_sdb(params = {})
+    RightAws::ActiveSdb.establish_connection key, secret, connection_params(params)
+  end
+
+  def sss(params = {})
+    return @s3 if @s3
+    @s3 = RightAws::S3.new(key, secret, connection_params(params))
+    unless @s3
+      puts "Amazon cannot connect to S3"
+      exit 3
+    end
+    @s3
+  end
+
+  def sqs(params = {})
+    return @sqs if @sqs
+    @sqs = RightAws::SqsGen2.new(key, secret, connection_params(params))
+    unless @sqs
+      puts "Amazon cannot connect to S3"
+      exit 3
+    end
+    @sqs
+  end
+
+  def elb(params = {})
+    return @elb if @elb
+    @elb = RightAws::ElbInterface.new(key, secret, connection_params(params))
+    unless @elb
+      puts "Amazon cannot connect to elb"
+      exit 3
+    end
+    @elb
+  end
+
+  def ecc(params = {})
+    return @ec2 if @ec2
+    @ec2 = RightAws::Ec2.new(key, secret, connection_params(params))
+    unless @ec2
+      puts "Amazon cannot connect to EC2"
+      exit 3
+    end
+    @ec2
+  end
+
+  def domains(reload = false)
+    @sdb_domains = if !@sdb_domains or reload
+                     sdb.list_domains[:domains]
+                   else
+                     @sdb_domains
+                   end
+  end
+
+  def in_cluster?
+    return @check unless @check.nil?
+    check = false
+    begin
+      Timeout::timeout(2) do
+        begin
+          s = TCPSocket.new('169.254.169.254', 80)
+          s.close
+          check = true
+        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+          # NOP
+        end
+      end
+    rescue Timeout::Error
+    end
+
+    @check = check
+  end
+
+  def connection_params(params = {})
+    params = {:multi_thread => true} if params.empty?
+    unless params.include?(:logger) 
+      params.merge(:logger => logger)
+    else
+      params
+    end
+  end
+
+  def store(key, io)
+    bucket.put(key, io)
+  end
+
+  def retrieve(key)
+    bucket.get key
+  end
+
+  def save_credentials(creds, key = nil)
+    key ||= @options.bucket_key
+    bucket.put(key, creds)
+  end
+
+  def save_images(input)
+    key ||= @options.cluster_image_key
+    bucket.put key, input, {}, 'public-read'
+  end
+
+  def credentials_url(seconds = 1200)
+    keygen = RightAws::S3Generator::Key.new(bucket, @options.bucket_key)
+    keygen.get(seconds)
+  end
+
+  def period(args)
+    shots = {}
+    ecc.describe_snapshots.each do |shot|
+      next unless shot[:aws_owner].to_s == @options.owner.to_s
+      next unless shot[:aws_status] == 'completed'
+      shot[:started_at] = Time.parse shot[:aws_started_at]
+      aid = shot[:aws_volume_id]
+      if shots.include? aid
+        shots[aid].push shot
+      else
+        shots.merge! aid => [shot]
+      end
+    end
+
+    stores = @options.volumes['stores'] || []
+    defaults = if @options.volumes.include?('defaults')
+                 @options.volumes['defaults']
+               else
+                 {}
+               end
+
+    shots.keys.each do |k|
+      opts = if v = stores.detect {|v| v['aws_id'] == k} 
+               defaults.merge v
+             else
+               defaults
+             end
+      snap_count = opts['snaps'] || 2
+      snap_count = 2 unless snap_count > 1
+
+      ordered = shots[k].sort {|a,b| a[:started_at] <=> b[:started_at] }.reverse
+      ordered.slice(snap_count, ordered.length).map {|shot|
+        ecc.delete_snapshot shot[:aws_id]
+        shot[:aws_id]
+      }
+    end
+  end
+
+  def bucket
+    unless @options.cluster_bucket
+      puts "#{Cluster::NAME} has not been configured with a bucket for client materials."
+      exit 2
+    end
+
+    @bucket ||= sss.bucket(@options.cluster_bucket, true)
+
+    unless @bucket
+      puts "#{Cluster::NAME} bucket named #{@options.cluster_bucket} cannot be created or accessed."
+      exit 2
+    else
+      @bucket
+    end
+  end
+
+  def get_image_file
+    require 'open-uri'
+    open(Cluster::IMAGES) {|f| YAML::load(f) }
+  end
+
+  def get_image(bits)
+    @image_file ||= get_image_file
+    case bits
+    when 32
+      @image_file['thirtytwo']
+    when 64
+      @image_file['sixtyfour']
+    end
+  end
+
+  def options
+    @options
+  end
+
+  def security(groups)
+    ecc.describe_security_groups(groups).inject({}) {|m,g|
+      p = g[:aws_perms].map {|p|
+        next unless p[:cidr_ips]
+        [p[:cidr_ips], p[:from_port]..p[:to_port]]
+      }.compact
+
+      m.merge g[:aws_group_name] => p
+    }
+  end
+
+  def revoke(ips)
+    ips.map {|ip|
+      ecc.revoke_security_group_IP_ingress('access', 22, 22, 'tcp', ip) and ip
+    }
+  end
+
+  def authorize(ips)
+    ips.map {|ip|
+      ecc.authorize_security_group_IP_ingress('access', 22, 22, 'tcp', ip) and ip
+    }
+  end
+
+  def query(qry)
+    res = sdb.select qry
+    if res
+      self.class.from_sdb_results res
+    else
+      nil
+    end
+  end
+
+
+  class << self
+    def to_sdb_attributes(args)
+      attrs = {}
+      args.each do |k, v|
+        v && attrs.merge!(k => Array(v))
+      end
+      attrs
+    end
+
+    def from_sdb_results(res)
+      res[:items].inject([]) {|m, obj|
+        aws_id = obj.keys.first
+        attrs = obj[aws_id]
+        args = attrs.keys.inject({:aws_id => aws_id}) {|n, attr|
+          val = attrs[attr]
+          if val.empty?
+            n.merge attr => nil
+          elsif val.length == 1
+            n.merge attr => val.first
+          else
+            n.merge attr => val
+          end
+        }
+        
+        m << args
+      }
+    end
+  end
+end