auser-poolparty 0.0.8 → 0.0.9

data/lib/modules/ec2_wrapper.rb

@@ -16,9 +16,11 @@ module PoolParty
           :maxCount => 1,
           :key_name => Application.keypair,
           :size => "#{Application.size}")
-          
-        item = instance.RunInstancesResponse.instancesSet.item
-        EC2ResponseObject.get_hash_from_response(item)
+        begin
+          item = instance#.instancesSet.item
+          EC2ResponseObject.get_hash_from_response(item)
+        rescue Exception => e          
+        end
       end
       # Shutdown the instance by instance_id
       def terminate_instance!(instance_id)
@@ -29,9 +31,9 @@ module PoolParty
       end
       # Instance description
       def describe_instance(id)
-        instance = ec2.describe_instances(:instance_id => id)
-        item = instance.DescribeInstancesResponse.reservationSet.item.instancesSet.item
-        EC2ResponseObject.get_hash_from_response(item)
+        # instance = ec2.describe_instances(:instance_id => id)
+        # item = instance.reservationSet.item.first.instancesSet.item.first
+        EC2ResponseObject.get_hash_from_response(ec2.describe_instances(:instance_id => id))
       end
       # Get instance by id
       def get_instance_by_id(id)
@@ -55,28 +57,51 @@ module PoolParty
   end
   # Provides a simple class to wrap around the amazon responses
   class EC2ResponseObject
-    def self.get_descriptions(resp)
-      rs = resp.DescribeInstancesResponse.reservationSet.item
-      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+    def self.get_descriptions(resp)      
+      rs = get_response_from(resp)
+      
+      # puts rs.methods.sort - rs.ancestors.methods
       out = begin
-        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
+        if rs.respond_to?(:instancesSet)
+          [EC2ResponseObject.get_hash_from_response(rs.instancesSet.item)]
+        else
+          rs.collect {|r| 
+            if r.instancesSet.item.class == Array
+              r.instancesSet.item.map {|t| EC2ResponseObject.get_hash_from_response(t)}
+            else
+              [EC2ResponseObject.get_hash_from_response(r.instancesSet.item)]
+            end            
+          }.flatten.reject {|a| a.nil? }
+        end
       rescue Exception => e
-        begin
-          # Really weird bug with amazon's ec2 gem
-          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
-        rescue Exception => e
-          []
-        end                
+        # Really weird bug with amazon's ec2 gem
+        rs.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil? } rescue []
       end
+      
       out
     end
+    def self.get_response_from(resp)
+      begin
+        rs = resp.reservationSet.item unless resp.reservationSet.nil?
+        rs ||= resp.DescribeInstancesResponse.reservationSet.item
+        rs ||= rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+        rs.reject! {|a| a.nil? || a.empty? }
+      rescue Exception => e
+      end
+      rs
+    end
     def self.get_hash_from_response(resp)
-      {
-        :instance_id => resp.instanceId, 
-        :ip => resp.dnsName, 
-        :status => resp.instanceState.name, 
-        :launching_time => resp.launchTime
-      } rescue nil
+      begin
+        {
+          :instance_id => resp.instanceId, 
+          :ip => resp.dnsName, 
+          :status => resp.instanceState.name,
+          :launching_time => resp.launchTime,
+          :keypair => resp.keyName
+        }        
+      rescue Exception => e
+        nil
+      end      
     end
   end
 end

data/lib/modules/file_writer.rb

@@ -0,0 +1,38 @@
+module PoolParty
+  module FileWriter
+    def write_to_file_for(f="haproxy", node=nil, str="", &block)      
+      make_base_directory
+      File.open("#{base_tmp_dir}/#{node ? "#{node.name}-" : ""}#{f}", "w+") do |file|
+        file << str
+        file << block.call if block_given?
+      end
+    end
+    # Write a temp file with the content str
+    def write_to_temp_file(str="")        
+      tempfile = Tempfile.new("#{base_tmp_dir}/poolparty-#{rand(1000)}-#{rand(1000)}")
+      tempfile.print(str)
+      tempfile.flush
+      tempfile
+    end
+    def with_temp_file(str="", &block)
+      Tempfile.open "#{base_tmp_dir}/poolparty-#{rand(10000)}" do |fp|
+        fp.puts str
+        fp.flush
+        block.call(fp)
+      end
+    end
+    
+    def base_tmp_dir
+      File.join(user_dir, "tmp")
+    end
+    def remote_base_tmp_dir
+      "~/tmp"
+    end    
+    def make_base_directory
+      `mkdir -p #{base_tmp_dir}` unless File.directory?(base_tmp_dir)
+    end
+    def clear_base_directory
+      `rm -rf #{base_tmp_dir}/*` if File.directory?(base_tmp_dir)
+    end
+  end
+end

data/lib/modules/sprinkle_overrides.rb

@@ -0,0 +1,32 @@
+require "sprinkle"
+module Sprinkle
+  module Installers
+    class Source < Installer
+
+      def custom_dir(dir)
+        @custom_dir = dir
+      end
+      
+      def base_dir
+        if @custom_dir
+          return @custom_dir
+        elsif @source.split('/').last =~ /(.*)\.(tar\.gz|tgz|tar\.bz2|tb2)/
+          return $1
+        end
+        raise "Unknown base path for source archive: #{@source}, please update code knowledge"
+      end
+    end
+    
+    class Gem < Installer
+      protected
+        def install_sequence
+          cmd = "gem install -y #{gem}"
+          cmd << " --version '#{version}'" if version
+          cmd << " --source #{source}" if source
+          cmd
+        end
+    end
+    
+    
+  end
+end

data/lib/modules/vlad_override.rb

@@ -1,12 +1,13 @@
 require "vlad"
-class Rake::RemoteTask < Rake::Task
+class Rake::RemoteTask < Rake::Task  
   def run command
-    cmd = [ssh_cmd, ssh_flags, target_host, command].compact
+    cmd = [ssh_cmd, ssh_flags, target_host].compact
     result = []
     
-    warn cmd.join(' ') if $TRACE
+    commander = cmd.join(" ") << " \"#{command}\""
+    warn commander if $TRACE
 
-    pid, inn, out, err = popen4(*cmd.join(" "))
+    pid, inn, out, err = popen4(commander)
 
     inn.sync   = true
     streams    = [out, err]

data/lib/poolparty.rb

@@ -8,19 +8,20 @@ require 'rubygems'
 require "aws/s3"
 require "sqs"
 require "EC2"
+require "aska"
+require 'sprinkle'
+
 require 'thread'
 require "pp"
 require "tempfile"
-require "aska"
-require "vlad"
+
 begin
   require 'fastthread'
-  require 'thin'
   require 'system_timer'
-  Timer = SystemTimer  
+  @@timer = SystemTimer
 rescue LoadError
   require 'timeout'
-  Timer = Timeout
+  @@timer = Timeout
 end
 
 ## Load PoolParty
@@ -41,11 +42,14 @@ end
 
 module PoolParty
   module Version #:nodoc:
-    MAJOR = 0
-    MINOR = 0
-    TINY  = 8
+    @major = 0
+    @minor = 0
+    @tiny  = 9
 
-    STRING = [MAJOR, MINOR, TINY].join('.')
+    STRING = [@major, @minor, @tiny].join('.')
+  end
+  def timer
+    @@timer
   end
   # PoolParty options
   def options(opts={})
@@ -83,7 +87,7 @@ module PoolParty
     end
   end
   def load_plugins
-    Dir["#{plugin_dir}/**/init.rb"].each {|a| require a}
+    Dir["#{plugin_dir}/**/init.rb"].each {|a| require a} if File.directory?(plugin_dir)
   end
   def reset!
     @@installed_plugins = nil

data/lib/poolparty/application.rb

@@ -17,17 +17,21 @@ module PoolParty
         loading_options = opts.delete(:optsparse) || {}
         loading_options.merge!( {:argv => opts.delete(:argv)} )
         
-        load_options!(loading_options)
-        # default_options.merge!(opts)
+        config_file_location = (default_options[:config_file] || opts[:config_file])        
         # If the config_file options are specified and not empty
-        unless default_options[:config_file].nil? || default_options[:config_file].empty?
+        unless config_file_location.nil? || config_file_location.empty?
           require "yaml"
           # Try loading the file if it exists
-          filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
-          default_options.merge!( YAML.load(filedata) ) if filedata # We want the command-line to overwrite the config file
+          filedata = File.open("#{config_file_location}").read if File.file?("#{config_file_location}")
+          # We want the command-line to overwrite the config file
+          default_options.merge!( YAML.load(filedata) ) if filedata
         end
         
-        OpenStruct.new(default_options.merge(opts))
+        default_options.merge!(opts)
+        load_options!(loading_options) # Load command-line options        
+        default_options.merge!(local_user_data) unless local_user_data == {}
+        
+        OpenStruct.new(default_options)
       end
 
       # Load options via commandline
@@ -48,15 +52,16 @@ module PoolParty
           op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }          
           op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
           op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }          
-          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
+          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:environment] = env }
           op.on('-a address', '--public-ip address', "Associate this public address with the master node") {|s| default_options[:public_ip] = s}
           op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
-          op.on('-n name', '--name name', "Application name") {|n| default_options[:app_name] = n}
+          op.on('-a name', '--name name', "Application name") {|n| default_options[:app_name] = n}
           op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
           op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
+          op.on('-i', '--install-on-boot', 'Install the PoolParty and custom software on boot (default: false)') {|b| default_options[:install_on_load] = true}
           op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
           op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = true}
-          op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
+          op.on('-n number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
           op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x.to_i}
           
           op.on_tail("-V", "Show version") do
@@ -103,24 +108,33 @@ module PoolParty
           :keypair => ENV["KEYPAIR_NAME"],
           :ami => 'ami-4a46a323',
           :shared_bucket => "",
-          :services => "",
           :expand_when => "web_usage < 1.5\n memory > 0.85",
           :contract_when => "cpu < 0.20\n memory < 0.10",
           :os => "ubuntu",
           :plugin_dir => "vendor",
-          :install_on_load => true
+          :install_on_load => false
         }
       end
       # Services monitored by Heartbeat
-      # Always at least monitors haproxy
-      def managed_services
-        "#{services}"
-      end
       def master_managed_services
-        "cloud_master_takeover #{services}"
+        "cloud_master_takeover"
       end
+      alias_method :managed_services, :master_managed_services
       def launching_user_data
-        {:polling_time => polling_time}.to_yaml
+        {:polling_time => polling_time, 
+          :access_key => access_key, 
+          :secret_access_key => secret_access_key,
+          :user_data => user_data}.to_yaml
+      end
+      def local_user_data
+        @local_user_data ||= 
+        begin
+          @@timer.timeout(5.seconds) do
+            YAML.load(open("http://169.254.169.254/latest/user-data").read)
+          end
+        rescue Exception => e
+          {}
+        end
       end
       # Keypair path
       # Idiom:
@@ -156,8 +170,8 @@ module PoolParty
       def version
         PoolParty::Version::STRING
       end
-      def install_on_load?
-        options.install_on_load == true
+      def install_on_load?(bool=false)
+        options.install_on_load == true || bool
       end
       # Call the options from the Application
       def method_missing(m,*args)

data/lib/poolparty/master.rb

@@ -6,7 +6,10 @@ module PoolParty
     include Aska
     include Callbacks
     include Monitors
+    # ############################
     include Remoter
+    # ############################
+    include FileWriter
     
     def initialize
       super
@@ -24,6 +27,17 @@ module PoolParty
       message "Launching minimum_instances"
       launch_minimum_instances
       message "Waiting for master to boot up" 
+      
+      wait_for_all_instances_to_boot
+      
+      setup_cloud
+    end
+    def setup_cloud
+      install_cloud
+      configure_cloud
+    end
+    alias_method :start, :start!
+    def wait_for_all_instances_to_boot
       reset!
       while !number_of_pending_instances.zero?
         wait "2.seconds" unless Application.test?
@@ -34,20 +48,49 @@ module PoolParty
         message "Give some time for the instance ssh to start up"
         wait "15.seconds"
       end
-      install_cloud if Application.install_on_load?
-      configure_cloud
     end
-    alias_method :start, :start!
+    def wait_for_all_instances_to_terminate
+      reset!
+      while !list_of_terminating_instances.size.zero?
+        wait "2.seconds" unless Application.test?
+        waited = true
+        reset!
+      end
+      unless Application.test? || waited.nil?
+        message "Give some time for the instance ssh to start up"
+        wait "15.seconds"
+      end
+      reset!
+    end
+    # Configure the master because the master will take care of the rest after that
     def configure_cloud
       message "Configuring master"
-      master = get_node 0      
-      master.configure
-    end
-    def install_cloud
+      build_and_send_config_files_in_temp_directory
+      remote_configure_instances
+      
       Master.with_nodes do |node|
-        node.login_once
+        node.configure
       end
-      Provider.install_poolparty(nodes.collect {|a| a.ip })
+    end
+    def install_cloud(bool=false)
+      if Application.install_on_load? || bool
+        # Just in case, add the new ubuntu apt-sources as well as updating and fixing the 
+        # update packages.
+        update_apt_string =<<-EOE        
+          touch /etc/apt/sources.list
+          echo 'deb http://mirrors.cs.wmich.edu/ubuntu hardy main universe' >> /etc/apt/sources.list
+          sudo apt-get update --fix-missing
+        EOE
+        
+        execute_tasks do
+          ssh(update_apt_string)
+        end
+        Provider.install_poolparty(cloud_ips)
+        Provider.install_userpackages(cloud_ips)
+      end
+    end
+    def cloud_ips
+      @ips ||= nodes.collect {|a| a.ip }
     end
     # Launch the minimum number of instances. 
     def launch_minimum_instances
@@ -74,7 +117,6 @@ module PoolParty
     end
     alias_method :start_monitor, :start_monitor!
     def user_tasks
-      puts "in user_tasks"
     end
     # Sole purpose to check the stats, mainly in a plugin
     def check_stats
@@ -90,30 +132,85 @@ module PoolParty
     # This is a basic check against the local store of the instances that have the 
     # stack installed.
     def reconfigure_cloud_when_necessary
-      reconfigure_running_instances if number_of_unconfigured_nodes > 0
+      configure_cloud if number_of_unconfigured_nodes > 0
     end
-    alias_method :reconfiguration, :reconfigure_cloud_when_necessary
     def number_of_unconfigured_nodes
       nodes.reject {|a| a.stack_installed? }.size
     end
-    def grow_by_one
-      request_launch_new_instance      
-      self.class.get_master.configure
+    def grow_by(num=1)
+      request_launch_new_instances(num)
+      
+      wait_for_all_instances_to_boot
+      
+      reset!
+      configure_cloud
+    end
+    def shrink_by(num=1)
+      num.times do |i|
+        node = nodes.reject {|a| a.master? }[-1]
+        request_termination_of_instance(node.instance_id) if node        
+      end
+      wait_for_all_instances_to_terminate
+      configure_cloud
     end
-    def shrink_by_one
-      node = nodes.reject {|a| a.master? }[-1]
-      request_termination_of_instance(node.instance_id) if node
+    
+    def build_and_send_config_files_in_temp_directory
+      require 'ftools'
+      File.copy(get_config_file_for("cloud_master_takeover"), "#{base_tmp_dir}/cloud_master_takeover")
+      File.copy(get_config_file_for("heartbeat.conf"), "#{base_tmp_dir}/ha.cf")
+      
+      File.copy(Application.config_file, "#{base_tmp_dir}/config.yml") if Application.config_file && File.exists?(Application.config_file)
+      File.copy(Application.keypair_path, "#{base_tmp_dir}/keypair") if File.exists?(Application.keypair_path)
+      
+      File.copy(get_config_file_for("monit.conf"), "#{base_tmp_dir}/monitrc")
+      
+      copy_config_files_in_directory_to_tmp_dir("config/resource.d")
+      copy_config_files_in_directory_to_tmp_dir("config/monit.d")
+      
+      build_and_copy_heartbeat_authkeys_file
+      build_haproxy_file
+      Master.build_user_global_files
+        
+      Master.with_nodes do |node|
+        build_hosts_file_for(node)
+        build_reconfigure_instances_script_for(node)
+        Master.build_user_node_files_for(node)
+        
+        if Master.requires_heartbeat?
+          build_heartbeat_config_file_for(node)
+          build_heartbeat_resources_file_for(node)
+        end
+      end      
+    end
+    def cleanup_tmp_directory(c)
+      Dir["#{base_tmp_dir}/*"].each {|f| FileUtils.rm_rf f} if File.directory?("tmp/")
+    end
+    before :build_and_send_config_files_in_temp_directory, :cleanup_tmp_directory    
+    # Send the files to the nodes
+    def send_config_files_to_nodes(c)
+      run_array_of_tasks(rsync_tasks("#{base_tmp_dir}/*", "#{remote_base_tmp_dir}"))
+    end
+    after :build_and_send_config_files_in_temp_directory, :send_config_files_to_nodes
+    def remote_configure_instances
+      arr = []
+      Master.with_nodes do |node|
+        script_file = "#{remote_base_tmp_dir}/#{node.name}-configuration"
+        str=<<-EOC
+chmod +x #{script_file}
+/bin/sh #{script_file}
+        EOC
+        arr << "#{self.class.ssh_string} #{node.ip} '#{str.strip.runnable}'"
+      end
+      run_array_of_tasks(arr)
     end
     # Add an instance if the load is high
     def add_instance_if_load_is_high
-      request_launch_new_instance if expand?
+      grow_by(1) if expand?
     end
     alias_method :add_instance, :add_instance_if_load_is_high
     # Teardown an instance if the load is pretty low
-    def terminate_instance_if_load_is_low
-      if contract?
-        shrink_by_one
-      end
+    def terminate_instance_if_load_is_low      
+      shrink_by(1) if contract?
     end
     alias_method :terminate_instance, :terminate_instance_if_load_is_low
     # FOR MONITORING
@@ -129,42 +226,68 @@ module PoolParty
         node.restart_with_monit
       end
     end
-    # Reconfigure the running instances
-    # Since we are using vlad, running configure on one of the instances
-    # should configure all of the instances. We set the hosts in this file
-    def reconfigure_running_instances      
-      # nodes.each do |node|
-      #   node.configure if node.status =~ /running/
-      # end
-      master = get_node(0)
-      master.configure
-    end
     # Build the basic haproxy config file from the config file in the config directory and return a tempfile
     def build_haproxy_file
-      servers=<<-EOS
+      write_to_file_for("haproxy") do
+        servers=<<-EOS
 #{nodes.collect {|node| node.haproxy_entry}.join("\n")}
-      EOS
-      open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
-    end
-    # Build the hosts file and return a tempfile
-    def build_hosts_file
-      write_to_temp_file(nodes.collect {|a| a.hosts_entry }.join("\n"))
+        EOS
+        open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
+      end
     end
     # Build host file for a specific node
     def build_hosts_file_for(n)
-      servers=<<-EOS        
-#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}
-      EOS
-      servers
+      write_to_file_for("hosts", n) do
+        "#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}"
+      end
     end
     # Build the basic auth file for the heartbeat
-    def build_heartbeat_authkeys_file
-      write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read)
+    def build_and_copy_heartbeat_authkeys_file
+      write_to_file_for("authkeys") do
+        open(Application.heartbeat_authkeys_config_file).read
+      end
     end
     # Build heartbeat config file
     def build_heartbeat_config_file_for(node)
-      servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}"
-      open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers}
+      write_to_file_for("heartbeat", node) do
+        servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}" rescue ""
+        open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers}
+      end
+    end
+    def build_heartbeat_resources_file_for(node)
+      write_to_file_for("haresources", node) do
+        "#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}" rescue ""
+      end        
+    end
+    # Build basic configuration script for the node
+    def build_reconfigure_instances_script_for(node)
+      write_to_file_for("configuration", node) do
+        open(Application.sh_reconfigure_instances_script).read.strip ^ node.configure_tasks
+      end        
+    end
+    
+    # Try the user's directory before the master directory
+    def get_config_file_for(name)
+      if File.exists?("#{user_dir}/config/#{name}")
+        "#{user_dir}/config/#{name}"
+      else 
+        "#{root_dir}/config/#{name}"
+      end
+    end
+    # Copy all the files in the directory to the dest
+    def copy_config_files_in_directory_to_tmp_dir(dir)
+      dest_dir = "#{base_tmp_dir}/#{File.basename(dir)}"
+      FileUtils.mkdir_p dest_dir
+      
+      if File.directory?("#{user_dir}/#{dir}")        
+        Dir["#{user_dir}/#{dir}/*"].each do |file|
+          File.copy(file, dest_dir)
+        end
+      else
+        Dir["#{root_dir}/#{dir}/*"].each do |file|
+          File.copy(file, dest_dir)
+        end
+      end      
     end
     # Return a list of the nodes and cache them
     def nodes
@@ -172,6 +295,18 @@ module PoolParty
         RemoteInstance.new(inst.merge({:number => i}))
       end
     end
+    # Return a list of the nodes for each keypair and cache them
+    def cloud_nodes
+      @cloud_nodes ||= begin
+        nodes_list = []
+        cloud_keypairs.each {|keypair| 
+          list_of_nonterminated_instances(list_of_instances(keypair)).collect_with_index { |inst, i|
+            nodes_list << RemoteInstance.new(inst.merge({:number => i}))
+          }
+        }
+        nodes_list
+      end
+    end
     # Get the node at the specific index from the cached nodes
     def get_node(i=0)
       nodes.select {|a| a.number == i.to_i}.first
@@ -179,7 +314,7 @@ module PoolParty
     # Get the next node in sequence, so we can configure heartbeat to monitor the next node
     def get_next_node(node)
       i = node.number + 1
-      i = 0 if i >= (nodes.size - 1)
+      i = 0 if i >= nodes.size
       get_node(i)
     end
     # On exit command
@@ -195,10 +330,28 @@ module PoolParty
       end
       out
     end
+    def clouds_list
+      if number_of_all_pending_and_running_instances > 0
+        out = "-- ALL CLOUDS (#{number_of_all_pending_and_running_instances})--\n"
+        keypair = nil
+        out << cloud_nodes.collect {|node|
+          str = ""
+          if keypair != node.keypair
+            keypair = node.keypair;
+            str = "key pair: #{keypair} (#{number_of_pending_and_running_instances(keypair)})\n"
+          end
+          str += "\t"+node.description if !node.description.nil?
+        }.join("\n")
+      else
+        out = "Clouds are not running"
+      end
+      out
+    end
     # Reset and clear the caches
     def reset!
       @cached_descriptions = nil
       @nodes = nil
+      @cloud_nodes = nil
     end
     
     class << self
@@ -231,50 +384,11 @@ module PoolParty
       end
       # Build a heartbeat resources file from the config directory and return a tempfile
       def build_heartbeat_resources_file_for(node)
-        return nil unless node
-        "#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}"
-      end
-      # Build hosts files for a specific node
-      def build_hosts_file_for(node)
-        new.build_hosts_file_for(node)
-      end
-      # Build the scp script for the specific node
-      def build_scp_instances_script_for(node)
-        authkeys_file = write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read.strip)
-        if Master.requires_heartbeat?
-          ha_d_file =  Master.build_heartbeat_config_file_for(node)
-          haresources_file = Master.build_heartbeat_resources_file_for(node)
-        end
-        haproxy_file = Master.build_haproxy_file
-        hosts_file = Master.build_hosts_file_for(node)        
-                
-        str = open(Application.sh_scp_instances_script).read.strip ^ {
-            :cloud_master_takeover => "#{node.scp_string("#{root_dir}/config/cloud_master_takeover", "/etc/ha.d/resource.d/", :dir => "/etc/ha.d/resource.d")}",
-            :config_file => "#{node.scp_string(Application.config_file, "~/.config")}",
-            :authkeys => "#{node.scp_string(authkeys_file.path, "/etc/ha.d/authkeys", :dir => "/etc/ha.d/")}",
-            :resources => "#{node.scp_string("#{root_dir}/config/resource.d/*", "/etc/ha.d/resource.d/", {:switches => "-r"})}",
-            :monitrc => "#{node.scp_string(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")}",
-            :monit_d => "#{node.scp_string("#{File.dirname(Application.monit_config_file)}/monit/*", "/etc/monit.d/", {:switches => "-r", :dir => "/etc/monit.d/"})}",
-            :haproxy => "#{node.scp_string(haproxy_file, "/etc/haproxy.cfg")}",
-            
-            :ha_d => Master.requires_heartbeat? ? "#{node.scp_string(ha_d_file, "/etc/ha.d/ha.cf")}" : "",
-            :haresources => Master.requires_heartbeat? ? "#{node.scp_string(haresources_file, "/etc/ha.d/ha.cf")}" : "",
-            
-            :hosts => "#{node.scp_string(hosts_file, "/etc/hosts")}"
-          }
-        write_to_temp_file(str)
-      end
-      # Build basic configuration script for the node
-      def build_reconfigure_instances_script_for(node)
-        str = open(Application.sh_reconfigure_instances_script).read.strip ^ {
-          :config_master => "#{node.update_plugin_string}",
-          :start_pool_maintain => "pool maintain -c ~/.config -l ~/plugins",
-          :set_hostname => "hostname -v #{node.name}",
-          :start_s3fs => "/usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data"
-        }
-        write_to_temp_file(str)        
-      end
-      
+        return nil unless node && get_next_node(node)
+        new.write_to_file_for("haresources", node) do
+          "#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}"
+        end        
+      end      
       def set_hosts(c, remotetask=nil)
         unless remotetask.nil?
           rt = remotetask
@@ -309,20 +423,28 @@ module PoolParty
       EOS
       open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port}
       end
-      # Write a temp file with the content str
-      def write_to_temp_file(str="")
-        tempfile = Tempfile.new("pool-party-#{rand(1000)}-#{rand(1000)}")
-        tempfile.print(str)
-        tempfile.flush
-        tempfile
-      end
-      def with_temp_file(str="", &block)
-        Tempfile.open "pool-party-#{rand(10000)}" do |fp|
-          fp.puts str
-          fp.flush
-          block.call(fp)
+      
+      # Placeholders
+      def build_user_global_files
+        global_user_files.each do |arr|
+          write_to_file_for(arr[0]) &arr[1]
+        end
+      end
+      def build_user_node_files_for(node)
+        user_node_files.each do |arr|
+          write_to_file_for(arr[0], node) do
+            arr[1].call(node)
+          end
         end
       end
+      def define_global_user_file(name, &block)
+        global_user_files << [name, block]
+      end
+      def global_user_files;@global_user_files ||= [];end
+      def define_node_user_file(name, &block)
+        user_node_files << [name, block]
+      end
+      def user_node_files;@user_node_files ||= [];end
     end
     
   end