appscale-tools 1.6.0

data/lib/custom_exceptions.rb

@@ -0,0 +1,25 @@
+# Programmer: Chris Bunch
+
+# a generic class to represent exceptions thrown within AppScale
+class AppScaleException < Exception
+end
+
+# a class representing exceptions related to bad command line arguments
+# (see lib/parse_args)
+class BadCommandLineArgException < AppScaleException
+end
+
+# a class representing exceptions related to incorrectly configured
+# AppScale deployments
+class BadConfigurationException < AppScaleException
+end
+
+# a class representing exceptions related to cloud infrastructures
+# (e.g., if euca or ec2 throw errors)
+class InfrastructureException < AppScaleException
+end
+
+# a class representing exceptions related to app engine apps
+# to be uploaded
+class AppEngineConfigException < AppScaleException
+end

data/lib/encryption_helper.rb

@@ -0,0 +1,86 @@
+#!/usr/bin/ruby -w
+# Programmer: Chris Bunch
+
+require 'fileutils'
+require 'openssl'
+$:.unshift File.join(File.dirname(__FILE__), ".", "lib")
+require 'common_functions'
+
+module EncryptionHelper
+  def self.generate_secret_key(keyname="appscale")
+    path="~/.appscale/#{keyname}.secret"
+    secret_key = ""
+    possible = "0123456789abcdefghijklmnopqrstuvxwyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    possibleLength = possible.length
+    
+    32.times { |index|
+      secret_key << possible[rand(possibleLength)]
+    }
+    
+    full_path = File.expand_path(path)
+    File.open(full_path, "w") { |file|
+      file.puts(secret_key)
+    }
+    
+    return secret_key, path
+  end
+  
+  def self.generate_ssh_key(verbose, outputLocation, name, infrastructure, force)
+    ec2_output = ""
+    loop {
+      sleep(10)  # to avoid euca replay error message
+      ec2_output = CommonFunctions.shell("#{infrastructure}-add-keypair #{name} 2>&1")
+      break if ec2_output.include?("BEGIN RSA PRIVATE KEY")
+      if force
+        puts "Trying again. Saw this from #{infrastructure}-add-keypair: #{ec2_output}" if verbose
+        sleep(10)
+        delete_output = CommonFunctions.shell("#{infrastructure}-delete-keypair #{name} 2>&1")
+        puts "Saw this from #{infrastructure}-delete-keypair: #{delete_output}" if verbose
+      else
+        abort("The keyname you chose is already in the system. Please either run this tool again with the --force flag or run the following:\n#{infrastructure}-delete-keypair #{name}")
+      end
+    }
+    
+    # output is the ssh private key prepended with info we don't need
+    # delimited by the first \n, so rip it off first to get just the key
+    
+    #first_newline = ec2_output.index("\n")
+    #ssh_private_key = ec2_output[first_newline+1, ec2_output.length-1]
+
+    if outputLocation.class == String
+      outputLocation = [outputLocation]
+    end
+    
+    outputLocation.each { |path|
+      fullPath = File.expand_path(path)
+      File.open(fullPath, "w") { |file|
+        file.puts(ec2_output)
+      }
+      FileUtils.chmod(0600, fullPath) # else ssh won't use the key
+    }
+    
+    return
+  end
+  
+  def self.generate_pem_files(keyname)
+    key_loc = File.expand_path("~/.appscale/#{keyname}-key.pem")
+    cert_loc = File.expand_path("~/.appscale/#{keyname}-cert.pem")
+    
+    key = OpenSSL::PKey::RSA.generate(2048)
+    pub = key.public_key
+    ca = OpenSSL::X509::Name.parse("/C=US/ST=Foo/L=Bar/O=AppScale/OU=User/CN=appscale.cs.ucsb.edu/emailAddress=test@test.com")
+    cert = OpenSSL::X509::Certificate.new
+    cert.version = 2
+    cert.serial = Time.now.to_i
+    cert.subject = ca
+    cert.issuer = ca
+    cert.public_key = pub
+    cert.not_before = Time.now
+    cert.not_after = Time.now + 3600
+    cert.sign(key, OpenSSL::Digest::SHA1.new)
+    
+    File.open(key_loc, "w") { |f| f.write key.to_pem }
+    File.open(cert_loc, "w") { |f| f.write cert.to_pem }
+    return key_loc, cert_loc
+  end
+end

data/lib/godinterface.rb

@@ -0,0 +1,152 @@
+
+GOD_PORT = "17165"
+
+module GodInterface
+  def self.start_god(remote_ip, remote_key)
+    self.run_god_command("god &", remote_ip, remote_key)
+  end
+
+  def self.start(watch, start_cmd, stop_cmd, ports, env_vars=nil, remote_ip=nil, remote_key=nil)
+
+    ports = [ports] unless ports.class == Array
+
+    prologue = <<BOO
+    WATCH = "#{watch}"
+    START_CMD = "#{start_cmd}"
+    STOP_CMD = "#{stop_cmd}"
+    PORTS = [#{ports.join(', ')}]
+
+BOO
+
+    body = <<'BAZ'
+    PORTS.each do |port|
+      God.watch do |w|
+        w.name = "appscale-#{WATCH}-#{port}"
+        w.group = WATCH
+        w.interval = 30.seconds # default      
+        w.start = START_CMD
+        w.stop = STOP_CMD
+        w.start_grace = 20.seconds
+        w.restart_grace = 20.seconds
+        w.log = "/var/log/appscale/#{WATCH}-#{port}.log"
+        w.pid_file = "/var/appscale/#{WATCH}-#{port}.pid"
+    
+        w.behavior(:clean_pid_file)
+
+        w.start_if do |start|
+          start.condition(:process_running) do |c|
+            c.running = false
+          end
+        end
+    
+        w.restart_if do |restart|
+          restart.condition(:memory_usage) do |c|
+            c.above = 150.megabytes
+            c.times = [3, 5] # 3 out of 5 intervals
+          end
+    
+          restart.condition(:cpu_usage) do |c|
+            c.above = 50.percent
+            c.times = 5
+          end
+        end
+    
+        # lifecycle
+        w.lifecycle do |on|
+          on.condition(:flapping) do |c|
+            c.to_state = [:start, :restart]
+            c.times = 5
+            c.within = 5.minute
+            c.transition = :unmonitored
+            c.retry_in = 10.minutes
+            c.retry_times = 5
+            c.retry_within = 2.hours
+          end
+        end
+BAZ
+
+    if !env_vars.nil? and !env_vars.empty?
+      env_vars_str = ""
+
+      env_vars.each { |k, v|
+        env_vars_str += "          \"" + k + "\" => \"" + v + "\",\n"
+      }
+
+      body += <<BOO
+
+        w.env = {
+          #{env_vars_str}
+        }
+BOO
+    end
+
+    epilogue = <<BAZ
+      end
+    end
+BAZ
+
+    config_file = prologue + body + epilogue
+    tempfile = "/tmp/god-#{rand(10000)}.god"
+
+    CommonFunctions.write_file(tempfile, config_file)
+
+    if remote_ip
+      CommonFunctions.scp_file(tempfile, tempfile, remote_ip, remote_key)
+    end
+
+    if remote_ip
+      ip = remote_ip
+    else
+      ip = CommonFunctions.local_ip
+    end
+
+    #unless CommonFunctions.is_port_open?(ip, GOD_PORT, use_ssl=false)
+    #  self.run_god_command("god", remote_ip, remote_key)
+    #  sleep(5)
+    #end
+
+    self.run_god_command("god load #{tempfile}", remote_ip, remote_key)
+
+    sleep(5)
+
+    FileUtils.rm_f(tempfile)
+    if remote_ip
+      remove = "rm -rf #{tempfile}"
+      CommonFunctions.run_remote_command(ip, remove, remote_key, false)
+    end
+
+    #god_info = "Starting #{watch} on ip #{ip}, port #{ports.join(', ')}" +
+    #  " with start command [#{start_cmd}] and stop command [#{stop_cmd}]"
+    #puts god_info
+
+    self.run_god_command("god start #{watch}", remote_ip, remote_key)
+  end
+
+  def self.stop(watch, remote_ip=nil, remote_key=nil)
+    self.run_god_command("god stop #{watch}", remote_ip, remote_key)
+  end
+
+  def self.remove(watch, remote_ip=nil, remote_key=nil)
+    self.run_god_command("god remove #{watch}", remote_ip, remote_key)
+  end
+
+  def self.shutdown(remote_ip=nil, remote_key=nil)
+    %w{ uaserver pbserver memcached blobstore monitr loadbalancer }.each { |service|
+      self.run_god_command("god stop #{service}", remote_ip, remote_key)
+    }
+
+    self.run_god_command("god terminate", remote_ip, remote_key)
+  end
+
+  private
+  def self.run_god_command(cmd, ip, ssh_key)
+    local = ip.nil?
+    
+    if local
+      puts cmd
+    else
+      CommonFunctions.run_remote_command(ip, cmd, ssh_key, true)
+    end
+  end
+end
+

data/lib/node_layout.rb

@@ -0,0 +1,665 @@
+#!/usr/bin/ruby
+# Programmer: Jonathan Kupferman
+# Updated by Chris Bunch to add hybrid cloud support
+
+USED_SIMPLE_AND_ADVANCED_KEYS = "Used both simple and advanced layout roles." +
+  " Only simple (controller, servers) or advanced (master, appengine, etc) " +
+  "can be used"
+NO_INPUT_YAML_REQUIRES_MIN_IMAGES = "If no input yaml is specified, " +
+  "min_images must be specified."
+NO_INPUT_YAML_REQUIRES_MAX_IMAGES = "If no input yaml is specified, " +
+  "max_images must be specified."
+INPUT_YAML_REQUIRED = "An input yaml file is required for Xen, KVM, and " +
+  "hybrid cloud deployments"
+DUPLICATE_IPS = "You specified some IP addresses more than once, which is " +
+  "not allowed in simple deployments."
+NO_CONTROLLER = "No controller was specified"
+ONLY_ONE_CONTROLLER = "Only one controller is allowed"
+
+NODE_ID_REGEX = /(node|cloud(\d+))-(\d+)/
+DEFAULT_NUM_NODES = 1
+VALID_ROLES = [:master, :appengine, :database, :shadow, :open] + 
+  [:load_balancer, :login, :db_master, :db_slave, :zookeeper, :memcache] +
+  [:rabbitmq, :rabbitmq_master, :rabbitmq_slave]
+
+class NodeLayout
+  SIMPLE_FORMAT_KEYS = [:controller, :servers]
+  ADVANCED_FORMAT_KEYS = [:master, :database, :appengine, :open, :login, :zookeeper, :memcache, :rabbitmq]
+
+  # Required options are: database_type
+  def initialize(input_yaml, options, skip_replication=false)
+    @input_yaml = (input_yaml.kind_of?(String) ? YAML.load(input_yaml) : input_yaml)
+
+    @infrastructure = options[:infrastructure]
+    @database_type = options[:database]
+    @database_type = @database_type.to_sym if !@database_type.nil?
+    @min_images = options[:min_images]
+    @max_images = options[:max_images]
+    @replication = options[:replication]
+    @read_factor = options[:read_factor]
+    @write_factor = options[:write_factor]
+    
+    @nodes = []
+    @skip_replication = skip_replication
+  end
+
+  def valid?
+    if is_simple_format? 
+      valid_simple_format?[:result]
+    elsif is_advanced_format?
+      valid_advanced_format?[:result]
+    else
+      false
+    end
+  end
+
+  def errors
+    return [] if valid?
+
+    if is_simple_format? 
+      valid_simple_format?[:message]
+    elsif is_advanced_format?
+      valid_advanced_format?[:message]
+    elsif @input_yaml.nil?
+      [INPUT_YAML_REQUIRED]
+    else
+      keys = @input_yaml.keys
+
+      keys.each { |key|
+        if !(SIMPLE_FORMAT_KEYS.include?(key) || ADVANCED_FORMAT_KEYS.include?(key))
+          return ["The flag #{key} is not a supported flag"]
+        end
+      }
+
+      return [USED_SIMPLE_AND_ADVANCED_KEYS]
+    end
+  end
+
+  def is_simple_format?
+    if @input_yaml.nil?
+      if VALID_CLOUD_TYPES.include?(@infrastructure) and @infrastructure != "hybrid"
+        # When used with the cloud, the simple format doesn't require a yaml
+        # Note this is not so in the hybrid model - a yaml is required in
+        # that scenario.
+        return true
+      else
+        return false
+      end
+    end
+
+    @input_yaml.keys.each do |key|
+      return false if !SIMPLE_FORMAT_KEYS.include?(key)
+    end
+
+   true
+  end
+
+  def is_advanced_format?
+    return false if @input_yaml.nil?
+
+    @input_yaml.keys.each do |key|
+      return false if !ADVANCED_FORMAT_KEYS.include?(key)
+    end
+
+    true
+  end
+
+  def parse_ip(ip)
+    id, cloud = nil, nil
+
+    match = NODE_ID_REGEX.match(ip)
+    if match.nil?
+      id = ip
+      cloud = "not-cloud"
+    else
+      id = match[0]
+      cloud = match[1]
+    end
+
+    return id, cloud
+  end
+
+  def valid_simple_format?
+    # We already computed the nodes, its valid
+    # cgb: an optimization to ensure we don't keep calling this
+    # when it always returns the same thing anyways
+    return valid if !@nodes.empty?
+
+    if @input_yaml.nil?
+      if VALID_CLOUD_TYPES.include?(@infrastructure) and @infrastructure != "hybrid"
+        if @min_images.nil?
+          return invalid(NO_INPUT_YAML_REQUIRES_MIN_IMAGES)
+        end
+
+        if @max_images.nil?
+          return invalid(NO_INPUT_YAML_REQUIRES_MAX_IMAGES)
+        end
+
+        # No yaml was created so we will create a generic one and then allow it to be validated
+        @input_yaml = generate_cloud_layout
+      else
+        return invalid(INPUT_YAML_REQUIRED)
+      end
+    end
+
+    nodes = []
+    @input_yaml.each_pair do |role, ips|
+      next if ips.nil?
+
+      ips.each do |ip|
+        id, cloud = parse_ip(ip)
+        node = SimpleNode.new id, cloud, [role]
+
+        # In simple deployments the db master and rabbitmq master is always on
+        # the shadow node, and db slave / rabbitmq slave is always on the other
+        # nodes
+        is_master = node.is_shadow?
+        node.add_db_role @database_type, is_master
+        node.add_rabbitmq_role is_master
+
+        return invalid(node.errors.join(",")) if !node.valid?
+
+        if VALID_CLOUD_TYPES.include?(@infrastructure)
+          error_message = "Invalid cloud node ID: #{node.id} \n" +
+            "Cloud node IDs must be in the format 'node-{IDNUMBER}'" +
+            "\nor of the form cloud{CLOUDNUMBER}-{IDNUMBER} for hybrid deployments"
+          return invalid(error_message) if NODE_ID_REGEX.match(node.id.to_s).nil?
+        else
+          # Xen/KVM should be using the ip address as the node id
+          error_message = "Invalid virtualized node ID: #{node.id} \n" + 
+            "Virtualized node IDs must be a valid IP address"
+          return invalid(error_message) if IP_REGEX.match(node.id.to_s).nil?
+        end
+
+        nodes << node
+      end
+    end
+
+    # make sure that the user hasn't erroneously specified the same ip
+    # address more than once
+    all_ips = @input_yaml.values.flatten
+    duplicate_ips = all_ips.length - all_ips.uniq.length
+
+    unless duplicate_ips.zero?
+      return invalid(DUPLICATE_IPS)
+    end
+
+    if nodes.length == 1
+      # Singleton node should be master and app engine
+      nodes.first.add_role :appengine
+      nodes.first.add_role :memcache
+    end
+
+    # controller -> shadow
+    controller_count = 0
+    nodes.each do |node|
+      if node.is_shadow?
+        controller_count += 1
+      end
+    end
+
+    if controller_count == 0
+      return invalid(NO_CONTROLLER)
+    elsif controller_count > 1
+      return invalid(ONLY_ONE_CONTROLLER)
+    end
+
+    database_count = 0
+    nodes.each do |node|
+      if node.is_database?
+        database_count += 1
+      end
+    end
+
+    if @skip_replication
+      @nodes = nodes
+      return valid
+    end
+
+    rep = valid_database_replication? nodes
+    return rep unless rep[:result]
+
+    # Wait until it is validated to assign it
+    @nodes = nodes
+    valid
+  end
+
+  def valid_advanced_format?
+    # We already computed the nodes, its valid
+    return valid if !@nodes.empty?
+
+    node_hash = {}
+    @input_yaml.each_pair do |role, ips|
+      
+      ips.each_with_index do |ip, index|
+        node = nil
+        if node_hash[ip].nil?
+          id, cloud = parse_ip(ip)
+          node = AdvancedNode.new(id, cloud)
+        else
+          node = node_hash[ip]
+        end
+
+        if role.to_sym == :database
+          # The first database node is the master
+          is_master = index.zero?
+          node.add_db_role @database_type, is_master
+        elsif role.to_sym == :db_master
+          node.add_role :zookeeper
+          node.add_role role
+        elsif role.to_sym == :rabbitmq
+          # Like the database, the first rabbitmq node is the master
+          is_master = index.zero?
+          node.add_role :rabbitmq
+          node.add_rabbitmq_role is_master
+        else
+          node.add_role role
+        end
+        
+        node_hash[ip] = node
+      end
+    end
+
+    # Dont need the hash any more, make a nodes list
+    nodes = node_hash.values
+
+    nodes.each do |node|
+      return invalid(node.errors.join(",")) unless node.valid?
+
+      if VALID_CLOUD_TYPES.include?(@infrastructure)
+        error_message = "Invalid cloud node ID: #{node.id} \n" + 
+          "Cloud node ID must be in the format 'node-{IDNUMBER}'" +
+          "\nor of the form cloud{CLOUDNUMBER}-{IDNUMBER} for hybrid deployments"
+        return invalid(error_message) if NODE_ID_REGEX.match(node.id.to_s).nil?
+      else
+        # Xen/KVM should be using the ip address as the node id
+        error_message = "Invalid virtualized node ID: #{node.id} \n" + 
+          "Virtualized node IDs must be a valid IP address"
+        return invalid(error_message) if IP_REGEX.match(node.id.to_s).nil?
+      end
+    end
+
+    master_nodes = nodes.select { |node| node.is_shadow? }.compact
+
+    # need exactly one master
+    if master_nodes.length == 0
+      return invalid("No master was specified")
+    elsif master_nodes.length > 1
+      return invalid("Only one master is allowed")
+    end
+
+    master_node = master_nodes.first
+
+    login_node = nodes.select { |node| node.is_login? }.compact
+    # If a login node was not specified, make the master into the login node
+    if login_node.empty?
+      master_node.add_role :login
+    end
+
+    appengine_count = 0
+    nodes.each do |node|
+      if node.is_appengine?
+        appengine_count += 1
+      end
+    end
+
+    if appengine_count < 1
+      return invalid("Not enough appengine nodes were provided.")
+    end
+
+    memcache_count = 0
+    nodes.each do |node|
+      if node.is_memcache?
+        memcache_count += 1
+      end
+    end
+
+    # if no memcache nodes were specified, make all appengine nodes
+    # into memcache nodes
+    if memcache_count < 1
+      nodes.each { |node|
+        node.add_role :memcache if node.is_appengine?
+      }
+    end
+
+    if VALID_CLOUD_TYPES.include?(@infrastructure)
+      # If min and max aren't specified, they default to the number of nodes in the system
+      @min_images ||= nodes.length
+      @max_images ||= nodes.length
+
+      # TODO: look into if that first guard is really necessary with the preceding lines
+
+      if @min_images && nodes.length < @min_images
+        return invalid("Too few nodes were provided, #{nodes.length} were specified but #{@min_images} was the minimum")
+      end
+ 
+      if @max_images && nodes.length > @max_images
+        return invalid("Too many nodes were provided, #{nodes.length} were specified but #{@max_images} was the maximum")
+      end
+    end
+
+    zookeeper_count = 0
+    nodes.each do |node|
+      if node.is_zookeeper?
+        zookeeper_count += 1
+      end
+    end
+    master_node.add_role :zookeeper if zookeeper_count.zero?
+
+    # If no rabbitmq nodes are specified, make the shadow the rabbitmq_master
+    rabbitmq_count = 0
+    nodes.each do |node|
+      if node.is_rabbitmq?
+        rabbitmq_count += 1
+      end
+    end
+    if rabbitmq_count.zero?
+      master_node.add_role :rabbitmq
+      master_node.add_role :rabbitmq_master
+    end
+
+    # Any node that runs appengine needs rabbitmq to dispatch task requests to
+    # It's safe to add the slave role since we ensure above that somebody
+    # already has the master role
+    nodes.each do |node|
+      if node.is_appengine? and !node.is_rabbitmq?
+        node.add_role :rabbitmq_slave
+      end
+    end
+
+    database_count = 0
+    nodes.each do |node|
+      if node.is_database?
+        database_count += 1
+      end
+    end
+
+    if @skip_replication
+      @nodes = nodes
+      return valid
+    end
+
+    rep = valid_database_replication? nodes
+    return rep unless rep[:result]
+
+    # Wait until it is validated to assign it
+    @nodes = nodes
+
+    return valid
+  end
+
+  def valid_database_replication? nodes
+    database_node_count = 0
+    nodes.each do |node|
+      if node.is_database? or node.is_db_master?
+        database_node_count += 1
+      end
+    end
+
+    if database_node_count.zero?
+      return invalid("At least one database node must be provided.")
+    end
+
+    if @replication.nil?
+      if database_node_count > 3
+        # If there are a lot of database nodes, we default to 3x replication
+        @replication = 3
+      else
+        # If there are only a few nodes, replicate to each one of the nodes
+        @replication = database_node_count
+      end
+    end
+
+    if @replication > database_node_count
+      return invalid("The provided replication factor is too high. The replication factor (-n flag) cannot be greater than the number of database nodes.")
+    end
+
+    # Perform all the database specific checks here
+    if @database_type == :mysql && database_node_count % @replication != 0
+      return invalid("MySQL requires that the amount of replication be divisible by the number of nodes (e.g. with 6 nodes, 2 or 3 times replication). You specified #{database_node_count} database nodes which is not divisible by #{@replication} times replication.")
+    end
+
+    if @database_type == :voldemort
+      @read_factor ||= @replication
+      @write_factor ||= @replication
+
+      if @read_factor > @replication
+        return invalid("The provided read factor is too high. The read factor (-r flag) cannot be greater than the replication factor.")
+      elsif @write_factor > @replication
+        return invalid("The provided write factor is too high. The write factor (-w flag) cannot be greater than the replication factor.")
+      end
+    end
+
+    if @database_type == :simpledb
+      if ENV['SIMPLEDB_ACCESS_KEY'].nil?
+        return invalid("SimpleDB deployments require that the environment variable SIMPLEDB_ACCESS_KEY be set to your AWS access key.")
+      end
+
+      if ENV['SIMPLEDB_SECRET_KEY'].nil?
+        return invalid("SimpleDB deployments require that the environment variable SIMPLEDB_SECRET_KEY be set to your AWS secret key.")
+      end
+    end
+    
+    valid
+  end
+
+  # Generates an yaml file for non-hybrid cloud layouts which don't have them
+  def generate_cloud_layout
+    layout = {:controller => "node-0"}
+    servers = []
+    num_slaves = @min_images - 1
+    num_slaves.times do |i|
+      servers << "node-#{i+1}"
+    end
+
+    layout[:servers] = servers
+    YAML.load(layout.to_yaml)
+  end
+
+  def replication_factor
+    return nil unless valid?
+
+    @replication
+  end
+
+  # TODO: can we just replace the if w/ unless and change ! to = ?
+  # or does that not exactly work due to the || ?
+
+  def read_factor
+    return nil if !valid? || @database_type != :voldemort
+
+    @read_factor
+  end
+
+  def write_factor
+    return nil if !valid? || @database_type != :voldemort
+
+    @write_factor
+  end
+
+  def min_images
+    return nil unless valid? 
+
+    @min_images
+  end
+
+  def max_images
+    return nil unless valid?
+    
+    @max_images
+  end
+
+  def nodes
+    return [] unless valid?
+    
+    # Since the valid? check has succeded @nodes has been initialized
+    @nodes
+  end
+  
+  # head node -> shadow
+  def head_node
+    return nil unless valid?
+
+    head_node = @nodes.select { |n| n.is_shadow? }.compact
+    
+    # TODO: is the last guard necessary?
+    head_node.empty? ? nil : head_node[0]
+  end
+
+  def other_nodes
+    return [] unless valid?
+
+    other_nodes = @nodes.select { |n| !n.is_shadow? }.compact
+    
+    other_nodes.empty? ? [] : other_nodes
+  end
+
+  def db_master
+    return nil unless valid?
+
+    db_master = @nodes.select { |n| n.is_db_master? }.compact
+    
+    db_master.empty? ? nil : db_master[0]
+  end
+
+  def login_node
+    return nil unless valid?
+
+    login = @nodes.select { |n| n.is_login? }.compact
+    
+    login.empty? ? nil : login[0]
+  end
+
+
+  def to_hash
+    result = {}
+    # Put all nodes except the head node in the hash
+    other_nodes.each do |node|
+      result[node.id] = node.roles.join(":")
+    end
+    result
+  end
+
+  private
+  def valid message=nil
+    { :result => true, :message => message }
+  end
+
+  def invalid message
+    { :result => false, :message => message }
+  end
+end
+
+class Node
+  attr_accessor :roles, :id, :cloud
+
+  def initialize id, cloud, roles=[]
+    # For Xen/KVM id is the public ip address
+    # For clouds, id is node-X since the ip is not known
+    @id = id
+    @cloud = cloud
+    @roles = roles.map { |r| r.to_sym }
+
+    expand_roles
+  end
+
+  def add_db_role db_type, is_master
+    if is_master
+      add_role :db_master
+      add_role :zookeeper
+    else
+      add_role :db_slave
+    end
+  end
+
+  def add_rabbitmq_role is_master
+    if is_master
+      add_role :rabbitmq_master
+    else
+      add_role :rabbitmq_slave
+    end
+  end
+
+  def add_role role
+    @roles << role.to_sym
+    expand_roles
+  end
+
+  VALID_ROLES.each do |role|
+    method = "is_#{role.to_s}?"
+    send :define_method, method do
+      @roles.include?(role)
+    end
+  end
+  
+  def valid?
+    self.errors.empty?
+  end
+
+  def errors
+    @roles.map { |r| "Invalid role: #{r}" if !VALID_ROLES.include?(r) }.compact
+  end
+
+  def expand_roles
+    error_msg = "Expand roles should never be called on a node type." + \
+      " All nodes should be either a SimpleNode or AdvancedNode"
+    raise RuntimeError error_msg
+  end
+end
+
+
+class SimpleNode < Node
+  private
+  def expand_roles
+    if @roles.include?(:controller)
+      @roles.delete(:controller)
+      @roles << :shadow
+      @roles << :load_balancer
+      @roles << :database
+      @roles << :memcache # the database needs memcache
+      @roles << :login
+      @roles << :zookeeper
+      @roles << :rabbitmq
+    end
+
+    # If they specify a servers role, expand it out to
+    # be database, appengine, and memcache
+    if @roles.include?(:servers)
+      @roles.delete(:servers)
+      @roles << :appengine
+      @roles << :memcache
+      @roles << :database
+      @roles << :load_balancer
+      @roles << :rabbitmq
+    end
+
+    @roles.uniq!
+  end
+end
+
+class AdvancedNode < Node
+  private
+  def expand_roles
+    # make sure that deleting here doesn't screw things up
+    if @roles.include?(:master)
+      @roles.delete(:master)
+      @roles << :shadow
+      @roles << :load_balancer
+      @roles << :zookeeper
+    end
+
+    if @roles.include?(:login)
+      @roles << :load_balancer
+    end
+
+    if @roles.include?(:appengine)
+      @roles << :load_balancer
+    end
+
+    if @roles.include?(:database)
+      @roles << :memcache
+    end
+
+    @roles.uniq!
+  end
+end