jtzemp-poolparty 0.1.2

data/lib/poolparty.rb

@@ -0,0 +1,131 @@
+=begin rdoc
+  The main file, contains the client and the server application methods
+=end
+$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
+
+$TRACE = true
+
+# rubygems
+require 'rubygems'
+require "aws/s3"
+require "EC2"
+require "aska"
+begin
+  require 'crafterm-sprinkle'
+rescue LoadError
+  require 'sprinkle'
+end
+require "pp"
+require "tempfile"
+
+begin
+  require 'fastthread'
+  require 'system_timer'
+  @@timer = SystemTimer
+rescue LoadError
+  require 'thread'
+  require 'timeout'
+  @@timer = Timeout
+end
+
+## Load PoolParty
+pwd = File.dirname(__FILE__)
+
+# Load the required files
+# If there is an init file, load that, otherwise
+# require all the files in each directory
+%w(core modules s3 helpers poolparty).each do |dir|  
+  Dir["#{pwd}/#{dir}"].each do |dir|
+    begin
+      require File.join(dir, "init")
+    rescue LoadError => e
+      Dir["#{pwd}/#{File.basename(dir)}/**"].each {|file| require File.join(dir, File.basename(file))}
+    end
+  end
+end
+
+module PoolParty
+  class Version #:nodoc:
+    @major = 0
+    @minor = 1
+    @tiny  = 2
+
+    def self.string
+      [@major, @minor, @tiny].join('.')
+    end
+  end
+  def timer
+    @@timer
+  end
+  # PoolParty options
+  def options(opts={})
+    Application.options(opts)
+  end
+  # Are we working in verbose-mode
+  def verbose?
+    options.verbose == true
+  end
+  # Send a message if we are in verbose-mode
+  def message(msg="")
+    puts "-- #{msg}" if verbose?
+  end
+  # Root directory of the application
+  def root_dir
+    File.expand_path(File.dirname(__FILE__) + "/..")
+  end
+  # User directory
+  def user_dir
+    Application.working_directory
+  end
+  # Write string to a tempfile
+  def write_to_temp_file(str="")
+    tempfile = Tempfile.new("rand#{rand(1000)}-#{rand(1000)}")
+    tempfile.print(str)
+    tempfile.flush
+    tempfile
+  end
+  def register_monitor(*names)
+    names.each do |name|
+      unless registered_monitor?(name)
+        PoolParty::Monitors.extend name
+      
+        PoolParty::Master.send :include, name::Master
+        PoolParty::RemoteInstance.send :include, name::Remote
+        
+        registered_monitors << name
+      end
+    end
+  end
+  def registered_monitor?(name); registered_monitors.include?(name); end
+  def registered_monitors; @@registered_monitors ||= [];end
+  
+  def load_app
+    load_monitors
+    load_plugins
+  end  
+  def load_monitors
+    loc = File.directory?("#{user_dir}/monitors") ? "#{user_dir}/monitors" : "#{root_dir}/lib/poolparty/monitors"
+    Dir["#{loc}/*"].each {|f| require f}
+  end
+  
+  def load_plugins
+    Dir["#{plugin_dir}/**/init.rb"].each {|a| require a} if File.directory?(plugin_dir)
+  end
+  def reset!
+    @@registered_monitors = nil
+    @@installed_plugins = nil
+  end
+  def plugin_dir
+    "#{user_dir}/plugins"
+  end
+  def read_config_file(filename)
+    return {} unless filename
+    YAML.load(open(filename).read)
+  end
+  def include_cloud_tasks
+    Tasks.new.define_tasks
+  end
+  
+  alias_method :tasks, :include_cloud_tasks
+  alias_method :include_tasks, :include_cloud_tasks
+end

data/lib/poolparty/application.rb

@@ -0,0 +1,199 @@
+=begin rdoc
+  Application
+  This handles user interaction
+=end
+module PoolParty
+  class Application
+    class << self
+      attr_accessor :verbose, :options
+      
+      # The application options
+      def options(opts={})
+        @options ||= make_options(opts)        
+      end
+      # Make the options with the config_file overrides included
+      # Default config file assumed to be at config/config.yml
+      def make_options(opts={})
+        loading_options = opts.delete(:optsparse) || {}
+        loading_options.merge!( opts || {})
+        
+        load_options!(loading_options) # Load command-line options
+        config_file_location = (default_options[:config_file] || opts[:config_file])
+        
+        # If the config_file options are specified and not empty
+        unless config_file_location.nil? || config_file_location.empty?
+          require "yaml"
+          # Try loading the file if it exists
+          filedata = File.open("#{config_file_location}").read if File.file?("#{config_file_location}")
+          default_options.merge!( YAML.load(filedata) ) if filedata rescue ""
+        end
+        # We want the command-line to overwrite the config file
+        default_options.merge!(local_user_data) unless local_user_data.nil?        
+        OpenStruct.new(default_options)
+      end
+
+      # Load options via commandline
+      def load_options!(opts={})
+        require 'optparse'
+        OptionParser.new do |op|
+          op.banner = opts[:banner] if opts[:banner]
+          op.on('-A key', '--access-key key', "Ec2 access key (ENV['AWS_ACCESS_KEY'])") { |key| default_options[:access_key] = key }
+          op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['AWS_SECRET_ACCESS'])") { |key| default_options[:secret_access_key] = key }
+          op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-40bc5829')") {|id| default_options[:ami] = id }
+          op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
+          op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }
+          # //THIS IS WHERE YOU LEFT OFF
+          op.on('-D working directory', '--dir dir', "Working directory") { |d| default_options[:working_directory] = d }
+          
+          op.on('--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
+          op.on('-r names', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
+          op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
+          op.on('-l plugin_dir', '--plugin-dir dir', "Plugin directory (default: '')") {|file| default_options[:plugin_dir] = file }
+          op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
+          op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }          
+          op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
+          op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }          
+          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:environment] = env }
+          op.on('-a address', '--public-ip address', "Associate this public address with the master node") {|s| default_options[:public_ip] = s}
+          op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
+          op.on('-a name', '--name name', "Application name") {|n| default_options[:app_name] = n}
+          op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
+          op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data.to_str }
+          op.on('-i', '--install-on-boot', 'Install the PoolParty and custom software on boot (default: false)') {|b| default_options[:install_on_load] = true}
+          op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
+          op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = true}
+          op.on('-n number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
+          op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x.to_i}
+          
+          op.on_tail("-V", "Show version") do
+            puts Application.version
+            exit
+          end
+          op.on_tail("-h", "-?", "Show this message") do
+            puts op
+            exit
+          end
+        end.parse!(opts[:argv] ? opts.delete(:argv) : ARGV.dup)
+      end
+      
+      # Basic default options
+      # All can be overridden by the command line
+      # or in a config.yml file
+      def default_options
+        @default_options ||= {
+          :app_name => "application_name",
+          :host_port => 80,
+          :client_port => 8001,
+          :environment => 'development',
+          :verbose => false,
+          :logging => true,
+          :size => "m1.small",
+          :polling_time => "30.seconds",
+          :user_data => "",
+          :heavy_load => 0.80,
+          :light_load => 0.15,
+          :minimum_instances => 2,
+          :maximum_instances => 4,
+          :public_ip => "",
+          :access_key => ENV["AWS_ACCESS_KEY"],
+          :secret_access_key => ENV["AWS_SECRET_ACCESS"],
+          :config_file => if ENV["CONFIG_FILE"] && !ENV["CONFIG_FILE"].empty?
+            ENV["CONFIG_FILE"]
+          elsif File.file?("config/config.yml")
+            "config/config.yml"
+          else
+            nil
+          end,
+          :username => "root",
+          :ec2_dir => (ENV["EC2_HOME"].nil? || ENV["EC2_HOME"].empty?) ? "~/.ec2" : ENV["EC2_HOME"],
+          :keypair => (ENV["KEYPAIR_NAME"].nil? || ENV["KEYPAIR_NAME"].empty?) ? File.basename(`pwd`).strip : ENV["KEYPAIR_NAME"],
+          :ami => 'ami-44bd592d',
+          :shared_bucket => "",
+          :expand_when => "web < 1.5\n memory > 0.85",
+          :contract_when => "cpu < 0.20\n memory < 0.10",
+          :os => "ubuntu",
+          :plugin_dir => "plugins",
+          :install_on_load => false,
+          :working_directory => Dir.pwd
+        }
+      end
+      # Services monitored by Heartbeat
+      def master_managed_services
+        "cloud_master_takeover"
+      end
+      alias_method :managed_services, :master_managed_services
+      def launching_user_data
+        hash_to_launch_with.to_yaml
+      end
+      def hash_to_launch_with
+        @hash ||= { :polling_time => polling_time, 
+          :access_key => access_key, 
+          :secret_access_key => secret_access_key,
+          :user_data => user_data,
+          :keypair => keypair,
+          :keypair_path => "/mnt"
+        }
+      end
+      def local_user_data 
+        @local_user_data ||= begin
+          @@timer.timeout(2.seconds) do
+            YAML.load(open("http://169.254.169.254/latest/user-data").read)
+          end
+        rescue Exception => e
+          {}
+        end
+      end
+      # For testing purposes
+      def reset!
+        @options = nil
+        @local_user_data = nil
+      end
+      # Keypair path
+      # Idiom:
+      #  /Users/username/.ec2/[name]
+      def keypair_path
+        options.keypair_path ? options.keypair_path : "#{ec2_dir}/#{keypair_name}"
+      end
+      def keypair_name
+        "id_rsa-#{keypair}"
+      end
+      # Are we in development or test mode
+      %w(development production test).each do |env|
+        eval <<-EOE
+          def #{env}?
+            environment == '#{env}'
+          end
+        EOE
+      end
+      def environment=(env)
+        environment = env
+      end
+      def maintain_pid_path
+        "/var/run/pool_maintain.pid"
+      end
+      %w(scp_instances_script reconfigure_instances_script).each do |file|
+        define_method "sh_#{file}" do
+          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.sh")
+        end
+      end
+      # Standard configuration files
+      %w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
+        define_method "#{file}_config_file" do
+          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
+        end
+      end
+      def version
+        PoolParty::Version.string
+      end
+      def install_on_load?(bool=false)
+        options.install_on_load == true || bool
+      end
+      # Call the options from the Application
+      def method_missing(m,*args)
+        options.methods.include?("#{m}") ? options.send(m,args) : super
+      end
+    end
+        
+  end
+    
+end

data/lib/poolparty/init.rb

@@ -0,0 +1,6 @@
+=begin rdoc
+  Load the files in order
+=end
+%w(optioner application thread_pool scheduler provider remoter remoting remote_instance master monitors tasks plugin).each do |f|
+  require File.join(File.dirname(__FILE__), f)
+end

data/lib/poolparty/master.rb

@@ -0,0 +1,492 @@
+=begin rdoc
+  The basic master for PoolParty
+=end
+module PoolParty
+  class Master < Remoting
+    include Aska
+    include Callbacks
+    # ############################
+    include Remoter
+    # ############################
+    include FileWriter
+    
+    def initialize
+      super            
+      
+      self.class.send :rules, :contract_when, Application.options.contract_when unless are_rules?(:contract_when)
+      self.class.send :rules, :expand_when, Application.options.expand_when unless are_rules?(:expand_when)
+    end
+    # Start the cloud
+    def start_cloud!
+      start!
+    end
+    alias_method :start_cloud, :start_cloud!
+    # Start the cloud, which launches the minimum_instances
+    def start!
+      message "Launching minimum_instances"
+      launch_minimum_instances
+      message "Waiting for master to boot up" 
+      
+      wait_for_all_instances_to_boot
+      
+      setup_cloud
+    end
+    def setup_cloud
+      install_cloud
+      configure_cloud
+    end
+    alias_method :start, :start!
+    def wait_for_all_instances_to_boot
+      reset!
+      while !number_of_pending_instances.zero?
+        wait "2.seconds" unless Application.test?
+        waited = true
+        reset!
+      end
+      unless Application.test? || waited.nil?
+        message "Give some time for the instance ssh to start up"
+        wait "15.seconds"
+      end
+    end
+    def wait_for_all_instances_to_terminate
+      reset!
+      while !list_of_terminating_instances.size.zero?
+        wait "2.seconds" unless Application.test?
+        waited = true
+        reset!
+      end
+      unless Application.test? || waited.nil?
+        message "Give some time for the instance ssh to start up"
+        wait "15.seconds"
+      end
+      reset!
+    end
+    # Configure the master because the master will take care of the rest after that
+    def configure_cloud
+      message "Configuring master"
+      build_and_send_config_files_in_temp_directory
+      remote_configure_instances
+      
+      nodes.each do |node|
+        node.configure
+      end      
+    end
+    before :install_cloud, :add_ssh_key
+    after :configure_cloud, :remove_ssh_key
+    def add_ssh_key(i)
+      Kernel.system("ssh-add #{Application.keypair_path} >/dev/null 2>/dev/null")
+    end
+    def remove_ssh_key(i)
+      Kernel.system("ssh-add -d #{Application.keypair_name} >/dev/null 2>/dev/null")
+    end
+    def install_cloud(bool=false)
+      if Application.install_on_load? || bool
+        # Just in case, add the new ubuntu apt-sources as well as updating and fixing the 
+        # update packages.
+        update_apt_string =<<-EOE        
+          touch /etc/apt/sources.list
+          echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
+          apt-get update --fix-missing
+        EOE
+        
+        ssh(update_apt_string)
+        
+        Provider.install_poolparty
+        
+        # For plugins
+        nodes.each do |node|
+          node.install
+        end
+        
+      end
+    end
+    def cloud_ips
+      @ips ||= nodes.collect {|a| a.ip }
+    end
+    # Launch the minimum number of instances. 
+    def launch_minimum_instances
+      request_launch_new_instances(Application.minimum_instances - number_of_pending_and_running_instances)
+      nodes
+    end
+    # Start monitoring the cloud with the threaded loop
+    def start_monitor!
+      begin
+        trap("INT") do
+          on_exit
+          exit
+        end
+        # Daemonize only if we are not in the test environment
+        run_thread_loop(:daemonize => !Application.test?) do
+          add_task {PoolParty.message "Checking cloud"}
+          add_task {launch_minimum_instances}
+          add_task {reconfigure_cloud_when_necessary}
+          add_task {scale_cloud!}
+          add_task {check_stats}
+        end
+      rescue Exception => e
+        Process.kill("HUP", Process.pid)
+      end
+    end
+    alias_method :start_monitor, :start_monitor!
+    def user_tasks
+    end
+    # Sole purpose to check the stats, mainly in a plugin
+    def check_stats
+      str = registered_monitors.collect {|m| "#{m}"}
+      PoolParty.message "Monitors: #{str.join(", ")}"
+    end
+    # Add an instance if the cloud needs one ore terminate one if necessary
+    def scale_cloud!
+      add_instance_if_load_is_high
+      terminate_instance_if_load_is_low
+    end
+    alias_method :scale_cloud, :scale_cloud!
+    # Tough method:
+    # We need to make sure that all the instances have the required software installed
+    # This is a basic check against the local store of the instances that have the 
+    # stack installed.
+    def reconfigure_cloud_when_necessary
+      PoolParty.message "#{number_of_unconfigured_nodes} unconfigured nodes"
+      configure_cloud if number_of_unconfigured_nodes > 0
+    end
+    def number_of_unconfigured_nodes
+      # TODO: Find a better way to tell if the nodes are configured.
+      nodes.reject {|a| a.stack_installed? }.size
+    end
+    def grow_by(num=1)
+      request_launch_new_instances(num)
+      
+      wait_for_all_instances_to_boot
+      
+      reset!
+      configure_cloud
+    end
+    def shrink_by(num=1)
+      num.times do |i|
+        # Get the last node that is not the master
+        node = nodes.reject {|a| a.master? }[-1]
+        res = request_termination_of_instance(node.instance_id) if node
+        PoolParty.message "#{res ? "Could" : "Could not"} shutdown instance"
+      end
+      wait_for_all_instances_to_terminate
+      configure_cloud
+    end
+    def make_base_tmp_dir(c)
+      `mkdir #{base_tmp_dir}` unless File.directory?(base_tmp_dir)
+    end
+    before :build_and_send_config_files_in_temp_directory, :make_base_tmp_dir
+    def build_and_send_config_files_in_temp_directory
+      require 'ftools'
+      if File.directory?(Application.plugin_dir)
+        Kernel.system("tar -czf #{base_tmp_dir}/plugins.tar.gz #{File.basename(Application.plugin_dir)}")
+      end
+      
+      if Master.requires_heartbeat?
+        build_and_copy_heartbeat_authkeys_file
+        File.copy(get_config_file_for("cloud_master_takeover"), "#{base_tmp_dir}/cloud_master_takeover")
+        File.copy(get_config_file_for("heartbeat.conf"), "#{base_tmp_dir}/ha.cf")
+      end
+      
+      File.copy(Application.config_file, "#{base_tmp_dir}/config.yml") if Application.config_file && File.exists?(Application.config_file)
+      File.copy(Application.keypair_path, "#{base_tmp_dir}/keypair") if File.exists?(Application.keypair_path)      
+      
+      copy_pem_files_to_tmp_dir
+            
+      copy_config_files_in_directory_to_tmp_dir("config/resource.d")
+      # copy_config_files_in_directory_to_tmp_dir("config/monit.d")
+            
+      build_haproxy_file
+      Master.build_user_global_files
+      
+      build_nodes_list
+        
+      Master.with_nodes do |node|
+        build_hosts_file_for(node)
+        build_reconfigure_instances_script_for(node)
+        Master.build_user_node_files_for(node)
+        
+        if Master.requires_heartbeat?
+          build_heartbeat_config_file_for(node)
+          build_heartbeat_resources_file_for(node)
+        end
+      end
+    end
+    def copy_pem_files_to_tmp_dir
+      %w(EC2_CERT EC2_PRIVATE_KEY).each do |key|
+        begin
+          file = `echo $#{key}`.strip
+          File.copy(file, "#{base_tmp_dir}/#{File.basename(file)}")
+        rescue Exception => e
+        end
+      end
+    end
+    def cleanup_tmp_directory(c)
+      Dir["#{base_tmp_dir}/*"].each {|f| FileUtils.rm_rf f} if File.directory?("tmp/")
+    end
+    before :build_and_send_config_files_in_temp_directory, :cleanup_tmp_directory
+    # Send the files to the nodes
+    def send_config_files_to_nodes(c)
+      run_array_of_tasks(rsync_tasks("#{base_tmp_dir}/*", "#{remote_base_tmp_dir}"))
+    end
+    after :build_and_send_config_files_in_temp_directory, :send_config_files_to_nodes
+    def remote_configure_instances
+      arr = []
+      Master.with_nodes do |node|
+        script_file = "#{remote_base_tmp_dir}/#{node.name}-configuration"
+        str=<<-EOC
+chmod +x #{script_file}
+/bin/sh #{script_file}
+        EOC
+        arr << "#{self.class.ssh_string} #{node.ip} '#{str.strip.runnable}'"
+      end
+      run_array_of_tasks(arr)
+    end
+    # Add an instance if the load is high
+    def add_instance_if_load_is_high
+      if expand?
+        PoolParty.message "Cloud needs expansion"
+        grow_by(1)
+      end
+    end
+    alias_method :add_instance, :add_instance_if_load_is_high
+    # Teardown an instance if the load is pretty low
+    def terminate_instance_if_load_is_low      
+      if contract?
+        PoolParty.message "Cloud to shrink"
+        shrink_by(1)
+      end
+    end
+    alias_method :terminate_instance, :terminate_instance_if_load_is_low
+    # FOR MONITORING
+    def contract?
+      valid_rules?(:contract_when)
+    end
+    def expand?
+      valid_rules?(:expand_when)
+    end
+    # Restart the running instances services with monit on all the nodes
+    def restart_running_instances_services
+      nodes.each do |node|
+        node.restart_with_monit
+      end
+    end
+    # Build the basic haproxy config file from the config file in the config directory and return a tempfile
+    def build_haproxy_file
+      write_to_file_for("haproxy") do
+        servers=<<-EOS
+#{nodes.collect {|node| node.haproxy_entry}.join("\n")}
+        EOS
+        open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
+      end
+    end
+    # Build host file for a specific node
+    def build_hosts_file_for(n)
+      write_to_file_for("hosts", n) do
+        "#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}"
+      end
+    end
+    def build_nodes_list
+      write_to_file_for(RemoteInstance.node_list_name) do
+        "#{cloud_ips.join("\n")}"
+      end
+    end
+    # Build the basic auth file for the heartbeat
+    def build_and_copy_heartbeat_authkeys_file
+      write_to_file_for("authkeys") do
+        open(Application.heartbeat_authkeys_config_file).read
+      end
+    end
+    # Build heartbeat config file
+    def build_heartbeat_config_file_for(node)
+      write_to_file_for("heartbeat", node) do
+        servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}" rescue ""
+        open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers}
+      end
+    end
+    def build_heartbeat_resources_file_for(node)
+      write_to_file_for("haresources", node) do
+        "#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}" rescue ""
+      end        
+    end
+    # Build basic configuration script for the node
+    def build_reconfigure_instances_script_for(node)
+      write_to_file_for("configuration", node) do
+        open(Application.sh_reconfigure_instances_script).read.strip ^ node.configure_tasks( !PoolParty.verbose? )
+      end        
+    end
+    
+    # Try the user's directory before the master directory
+    def get_config_file_for(name)
+      if File.exists?("#{user_dir}/config/#{name}")
+        "#{user_dir}/config/#{name}"
+      else
+        "#{root_dir}/config/#{name}"
+      end
+    end
+    # Copy all the files in the directory to the dest
+    def copy_config_files_in_directory_to_tmp_dir(dir)
+      dest_dir = "#{base_tmp_dir}/#{File.basename(dir)}"
+      FileUtils.mkdir_p dest_dir
+      
+      if File.directory?("#{user_dir}/#{dir}")        
+        Dir["#{user_dir}/#{dir}/*"].each do |file|
+          File.copy(file, dest_dir)
+        end
+      else
+        Dir["#{root_dir}/#{dir}/*"].each do |file|
+          File.copy(file, dest_dir)
+        end
+      end      
+    end
+    # Return a list of the nodes and cache them
+    def nodes
+      @nodes ||= list_of_nonterminated_instances.collect_with_index do |inst, i|
+        RemoteInstance.new(inst.merge({:number => i}))
+      end
+    end
+    # Return a list of the nodes for each keypair and cache them
+    def cloud_nodes
+      @cloud_nodes ||= begin
+        nodes_list = []
+        cloud_keypairs.each {|keypair| 
+          list_of_nonterminated_instances(list_of_instances(keypair)).collect_with_index { |inst, i|
+            nodes_list << RemoteInstance.new(inst.merge({:number => i}))
+          }
+        }
+        nodes_list
+      end
+    end
+    # Get the node at the specific index from the cached nodes
+    def get_node(i=0)
+      nodes.select {|a| a.number == i.to_i}.first
+    end
+    # Get the next node in sequence, so we can configure heartbeat to monitor the next node
+    def get_next_node(node)
+      i = node.number + 1
+      i = 0 if i >= nodes.size
+      get_node(i)
+    end
+    # On exit command
+    def on_exit      
+    end
+    # List the clouds
+    def list
+      if number_of_pending_and_running_instances > 0
+        out = "-- CLOUD (#{number_of_pending_and_running_instances})--\n"
+        out << nodes.collect {|node| node.description }.join("\n")
+      else
+        out = "Cloud is not running"
+      end
+      out
+    end
+    def clouds_list
+      if number_of_all_pending_and_running_instances > 0
+        out = "-- ALL CLOUDS (#{number_of_all_pending_and_running_instances})--\n"
+        keypair = nil
+        out << cloud_nodes.collect {|node|
+          str = ""
+          if keypair != node.keypair
+            keypair = node.keypair;
+            str = "key pair: #{keypair} (#{number_of_pending_and_running_instances(keypair)})\n"
+          end
+          str += "\t"+node.description if !node.description.nil?
+        }.join("\n")
+      else
+        out = "Clouds are not running"
+      end
+      out
+    end
+    # Reset and clear the caches
+    def reset!
+      @cached_descriptions = nil
+      @nodes = nil
+      @cloud_nodes = nil
+    end
+    
+    class << self
+      include PoolParty
+      include FileWriter
+            
+      def with_nodes(&block)
+        new.nodes.each &block
+      end
+      
+      def collect_nodes(&block)
+        new.nodes.collect &block
+      end
+      
+      def requires_heartbeat?
+        new.nodes.size > 1
+      end
+      def is_master_responding?
+        `ping -c1 -t5 #{get_master.ip}`
+      end
+      def get_master
+        new.nodes[0]
+      end
+      def cloud_ips
+        new.cloud_ips
+      end
+      def get_next_node(node)
+        new.get_next_node(node)
+      end
+      def set_hosts(c, remotetask=nil)
+        unless remotetask.nil?
+          rt = remotetask
+        end
+        
+        ssh_location = `which ssh`.gsub(/\n/, '')
+        rsync_location = `which rsync`.gsub(/\n/, '')
+        rt.set :user, Application.username
+        # rt.set :domain, "#{Application.user}@#{ip}"
+        rt.set :application, Application.app_name
+        rt.set :ssh_flags, "-i #{Application.keypair_path} -o StrictHostKeyChecking=no"
+        rt.set :rsync_flags , ['-azP', '--delete', "-e '#{ssh_location} -l #{Application.username} -i #{Application.keypair_path} -o StrictHostKeyChecking=no'"]
+        
+        master = get_master
+        rt.set :domain, "#{master.ip}" if master
+        Master.with_nodes { |node|
+          rt.host "#{Application.username}@#{node.ip}",:app if node.status =~ /running/
+        }
+      end
+            
+      def ssh_configure_string_for(node)
+        cmd=<<-EOC
+          #{node.update_plugin_string(node)}
+          pool maintain -c ~/.config -l #{PoolParty.plugin_dir}
+          hostname -v #{node.name}
+          /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
+        EOC
+      end
+      def build_haproxy_file
+      servers=<<-EOS
+#{collect_nodes {|node| node.haproxy_entry}.join("\n")}
+      EOS
+      open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
+      end
+      
+      # Placeholders
+      def build_user_global_files
+        global_user_files.each do |arr|
+          write_to_file_for(arr[0]) &arr[1]
+        end
+      end
+      def build_user_node_files_for(node)
+        user_node_files.each do |arr|
+          write_to_file_for(arr[0], node) do
+            arr[1].call(node)
+          end
+        end
+      end
+      def define_global_user_file(name, &block)
+        global_user_files << [name, block]
+      end
+      def global_user_files;@global_user_files ||= [];end
+      def define_node_user_file(name, &block)
+        user_node_files << [name, block]
+      end
+      def user_node_files;@user_node_files ||= [];end
+    end
+    
+  end
+end