dreadpiratepj-poolparty 0.0.8

data/lib/poolparty/remoter.rb

@@ -0,0 +1,106 @@
+=begin rdoc
+  Handle the remoting aspects of the remote_instances
+=end
+module PoolParty
+  module Remoter        
+    module ClassMethods
+      def ssh_string
+        "ssh -i #{Application.keypair_path} -o StrictHostKeyChecking=no -l #{Application.username}"
+      end
+    end
+    
+    module InstanceMethods      
+      include Callbacks
+      include Scheduler
+      
+      def scp local, remote, opts={}
+        data = open(local).read
+        begin
+          
+          cmd = "rsync --delete -azP -e '#{self.class.ssh_string} -l #{Application.username} -i #{Application.keypair_path}' "
+          
+          if opts[:dir]
+            ssh("mkdir -p #{opts[:dir]}")
+          end
+          
+          if opts[:single]
+            scp_tasks << "#{cmd} #{local} #{Application.username}@#{opts[:single]}:#{remote}"
+          else
+            target_hosts.each do |ip|
+              scp_tasks << "#{cmd} #{local} #{Application.username}@#{ip}:#{remote}"
+            end
+          end
+          
+        rescue Exception => e        
+        end      
+      end
+
+      def single_scp local, remote, opts={}
+        scp local, remote, opts.merge({:single => self.ip})
+      end
+
+      def ssh command="", opts={}, &block
+        cmd = self.class.ssh_string
+        if command.empty?
+          system("#{cmd} #{Application.username}@#{self.ip}")
+        else
+          if opts[:single]
+            ssh_tasks << "#{cmd} #{Application.username}@#{self.ip} '#{command.runnable}'"
+          else
+            target_hosts.each do |ip|
+              ssh_tasks << "#{cmd} #{Application.username}@#{ip} '#{command.runnable}'"
+            end
+          end
+        end
+      end
+      
+      def run_now command=""
+        system("#{self.class.ssh_string} #{Application.username}@#{self.ip} #{command.runnable}")
+      end
+      
+      def install *names
+        names.each {|name| install_tasks << name }
+      end
+      
+      def ssh_tasks;@ssh_tasks ||= [];end
+      def scp_tasks;@scp_tasks ||= [];end
+      def install_tasks;@install_tasks ||= [];end
+            
+      def reset!
+        @ssh_tasks = @scp_tasks = nil
+        @hosts = nil
+      end
+            
+      def execute_tasks(opts={})
+        # reset!
+        
+        set_hosts(nil) unless opts[:dont_set_hosts]
+        
+        yield if block_given?
+        
+        run_array_of_tasks(scp_tasks)
+        run_array_of_tasks(ssh_tasks)
+        
+        PoolParty.message "running #{ssh_tasks.size} tasks"
+      end
+      
+      def target_hosts
+        @hosts ||= Master.collect_nodes {|a| a.ip }
+      end
+      
+      def run_array_of_tasks(task_list)
+        add_task {`#{task_list.join(" && ")}`}
+        run_thread_list
+      end
+      
+      def set_hosts(c=nil)
+      end
+      
+    end
+  
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end

data/lib/poolparty/remoting.rb

@@ -0,0 +1,112 @@
+module PoolParty
+  extend self
+  
+  class Remoting
+    include PoolParty
+    include Ec2Wrapper
+    include Scheduler
+    
+    # == GENERAL METHODS    
+    # == LISTING
+    # List all the running instances associated with this account
+    def list_of_running_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /running/}
+    end
+    # Get a list of the pending instances
+    def list_of_pending_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /pending/}
+    end
+    # list of shutting down instances
+    def list_of_terminating_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /shutting/}
+    end
+    # list all the nonterminated instances
+    def list_of_nonterminated_instances
+      list_of_instances.reject {|a| a[:status] =~ /terminated/}
+    end
+    # List the instances, regardless of their states
+    def list_of_instances
+      get_instances_description
+    end
+    # Get number of pending instances
+    def number_of_pending_instances
+      list_of_pending_instances.size
+    end
+    # get the number of running instances
+    def number_of_running_instances
+      list_of_running_instances.size
+    end
+    # get the number of pending and running instances
+    def number_of_pending_and_running_instances
+      number_of_running_instances + number_of_pending_instances
+    end
+    # == LAUNCHING
+    # Request to launch a new instance
+    def request_launch_new_instance
+      if can_start_a_new_instance?
+        request_launch_one_instance_at_a_time
+        return true
+      else
+        return false
+      end
+    end
+    # Can we start a new instance?
+    def can_start_a_new_instance?
+      maximum_number_of_instances_are_not_running?
+    end
+    # Are the maximum number of instances running?
+    def maximum_number_of_instances_are_not_running?
+      list_of_running_instances.size < Application.maximum_instances
+    end
+    # Request to launch a number of instances
+    def request_launch_new_instances(num=1)
+      out = []
+      num.times {out << request_launch_one_instance_at_a_time}
+      out
+    end
+    # Launch one instance at a time
+    def request_launch_one_instance_at_a_time
+      reset!
+      while !number_of_pending_instances.zero?
+        wait "5.seconds"
+        reset!
+      end
+      return launch_new_instance!
+    end
+    # == SHUTDOWN
+    # Terminate all running instances
+    def request_termination_of_running_instances
+      list_of_running_instances.each {|a| terminate_instance!(a[:instance_id])}
+    end
+    # Request termination of all instances regardless of their state (includes pending instances)
+    def request_termination_of_all_instances
+      get_instances_description.each {|a| terminate_instance!(a[:instance_id])}
+    end
+    # Terminate instance by id
+    def request_termination_of_instance(id)
+      if can_shutdown_an_instance?
+        terminate_instance! id
+        return true
+      else
+        return false
+      end
+    end
+    # Can we shutdown an instance?
+    def can_shutdown_an_instance?
+      minimum_number_of_instances_are_running?
+    end
+    # Are the minimum number of instances running?
+    def minimum_number_of_instances_are_running?
+      list_of_running_instances.size > Application.minimum_instances
+    end
+    # Get the cached running_instances
+    def running_instances
+      @running_instances ||= update_instance_values
+    end
+    # Update the instance values
+    def update_instance_values
+      @running_instances = list_of_running_instances.collect {|a| RemoteInstance.new(a) }.sort
+    end
+  end
+    
+end

data/lib/poolparty/scheduler.rb

@@ -0,0 +1,103 @@
+module PoolParty
+  extend self
+  # Schedule tasks container
+  class ScheduleTasks
+    include ThreadSafeInstance
+    # Initialize tasks array and run
+    def tasks
+      @_tasks ||= []
+    end
+    # Synchronize the running threaded tasks
+    def run
+      unless tasks.empty?
+        self.class.synchronize do
+          tasks.reject!{|a|            
+            begin
+              a.run
+              a.join
+            rescue Exception => e    
+              puts "There was an error in the task: #{e} #{e.backtrace.join("\n")}"
+            end
+            true
+          }
+        end
+      end
+    end
+    # Add a task in a new thread
+    def <<(a, *args)
+      thread = Thread.new(a) do |task|
+        Thread.stop
+        Thread.current[:callee] = task
+        a.call args
+      end
+      tasks << thread
+    end
+    alias_method :push, :<<
+    # In the ThreadSafeInstance
+    make_safe :<<
+  end
+  # Scheduler class
+  module Scheduler
+    attr_reader :tasker
+    # Get the tasks or ScheduleTasks
+    def _tasker
+      @_tasker ||= ScheduleTasks.new
+    end
+    # Add a task to the new threaded block
+    def add_task(&blk)
+      _tasker.push proc{blk.call}
+    end
+    # Grab the polling_time
+    def interval
+      @interval ||= Application.polling_time
+    end
+    # Run the threads
+    def run_threads
+      _tasker.run
+    end
+    alias_method :run_tasks, :run_threads
+    # Daemonize the process
+    def daemonize
+      PoolParty.message "Daemonizing..."
+      
+      pid = fork do
+        Signal.trap('HUP', 'IGNORE') # Don't die upon logout
+        File.open("/dev/null", "r+") do |devnull|
+          $stdout.reopen(devnull)
+          $stderr.reopen(devnull)
+          $stdin.reopen(devnull) unless @use_stdin
+        end
+        yield if block_given?
+      end
+      Process.detach(pid)
+      pid
+    end
+    # Run the loop and wait the amount of time between running the tasks
+    # You can send it daemonize => true and it will daemonize
+    def run_thread_loop(opts={}, &block)
+      block = lambda {        
+        loop do
+          begin
+            run_thread_list(&block)
+            wait interval
+            reset!
+          rescue Exception => e
+            puts "There was an error in the run_thread_loop: #{e}"
+          end
+        end
+      }
+      # Run the tasks
+      opts[:daemonize] ? daemonize(&block) : block.call   
+    end
+    
+    def run_thread_list
+      yield if block_given?
+      run_threads
+    end
+    # Reset
+    def reset!
+      cached_variables.each {|cached| cached = nil }
+    end
+        
+  end
+end

data/lib/poolparty/tasks.rb

@@ -0,0 +1,29 @@
+module PoolParty
+  class Tasks
+    include Callbacks
+    
+    # Setup and define all the tasks
+    def initialize
+      yield self if block_given?
+    end
+    # Define the tasks in the rakefile
+    # From the rakefile
+    def define_tasks
+      # Run the command on the local system
+      def run(cmd)
+        system(cmd.runnable)
+      end
+      # Basic setup action
+      def setup_application
+        PoolParty.options({:config_file => (ENV["CONFIG_FILE"] || ENV["config"]) })
+      end
+      
+      # Require the poolparty specific tasks
+      Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].each { |t| eval open(t).read }
+      
+      Dir["#{PoolParty.plugin_dir}/*/Rakefile"].each {|t| load "#{t}" }
+      
+      true
+    end    
+  end
+end

data/lib/poolparty/tasks/cloud.rake

@@ -0,0 +1,57 @@
+# Cloud tasks
+namespace(:cloud) do
+  # Setup
+  task :init do
+    setup_application
+    raise Exception.new("You must specify your access_key and secret_access_key") unless Application.access_key && Application.secret_access_key
+  end
+  # Install the stack on all of the nodes
+  desc "Prepare all servers"
+  task :prepare => :init do
+    PoolParty::Master.new.nodes.each do |node|
+      node.install
+    end
+  end
+  # Start the cloud
+  desc "Start the cloud"
+  task :start => :init do
+    PoolParty::Master.new.start_cloud!
+  end
+  # Reload the cloud with the new updated data
+  desc "Reload all instances with updated data"
+  task :reload => :init do
+    PoolParty::Master.new.nodes.each do |node|
+      node.configure
+      node.restart_with_monit
+    end
+  end
+  # List the cloud
+  desc "List cloud"
+  task :list => :init do
+    PoolParty::Master.new.list
+  end
+  # Shutdown the cloud
+  desc "Shutdown the entire cloud"
+  task :shutdown => :init do
+    PoolParty::Master.new.request_termination_of_all_instances
+  end
+  # Watch the cloud and scale it if necessary
+  desc "Watch the cloud and maintain it"
+  task :scale => :init do
+    begin
+      PoolParty::Master.new.scale_cloud!
+    rescue Exception => e
+      puts "There was an error scaling the cloud: #{e}"
+    end
+    
+  end
+  # Maintain the cloud in a background process
+  desc "Maintain the cloud (run on the master)"
+  task :maintain => :init do
+    begin
+      PoolParty::Master.new.start_monitor!
+    rescue Exception => e
+      puts "There was an error starting the monitor: #{e}"
+    end
+  end
+end

data/lib/poolparty/tasks/development.rake

@@ -0,0 +1,38 @@
+namespace(:dev) do
+  task :init do
+    setup_application
+    run "mkdir ~/.ec2 >/dev/null 2>/dev/null" unless File.directory?("~/.ec2")      
+  end
+  # Setup a basic development environment for the user 
+  desc "Setup development environment specify the config_file"
+  task :setup => [:init] do
+    keyfilename = ".#{Application.keypair}_pool_keys"
+    run <<-EOR
+      echo 'export AWS_ACCESS_KEY_ID=\"#{Application.access_key}\"' > $HOME/#{keyfilename}
+      echo 'export AWS_SECRET_ACCESS_ID=\"#{Application.secret_access_key}\"' >> $HOME/#{keyfilename}
+      echo 'export EC2_HOME=\"#{Application.ec2_dir}\"' >> $HOME/#{keyfilename}
+      echo 'export KEYPAIR_NAME=\"#{Application.keypair}\"' >> $HOME/#{keyfilename}
+      echo 'export CONFIG_FILE=\"#{Application.config_file}\"' >> $HOME/#{keyfilename}
+      echo 'export EC2_PRIVATE_KEY=`ls ~/.ec2/#{Application.keypair}/pk-*.pem`;' >> $HOME/#{keyfilename}
+      echo 'export EC2_CERT=`ls ~/.ec2/#{Application.keypair}/cert-*.pem`;' >> $HOME/#{keyfilename}
+      source $HOME/#{keyfilename}
+    EOR
+  end
+  desc "Generate a keypair"
+  task :setup_keypair => :init do
+    unless File.file?(Application.keypair_path)
+      Application.keypair = "cloud"
+      run <<-EOR
+        ec2-add-keypair cloud > #{Application.keypair_path}
+        chmod 600 #{Application.keypair_path}
+      EOR
+    end
+  end
+  desc "Authorize base ports for application"
+  task :authorize_ports => :init do
+    run <<-EOR
+      ec2-authorize -p 22 default
+      ec2-authorize -p 80 default
+    EOR
+  end
+end