auser-poolparty 0.0.8

data/lib/poolparty/remoting.rb

@@ -0,0 +1,112 @@
+module PoolParty
+  extend self
+  
+  class Remoting
+    include PoolParty
+    include Ec2Wrapper
+    include Scheduler
+    
+    # == GENERAL METHODS    
+    # == LISTING
+    # List all the running instances associated with this account
+    def list_of_running_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /running/}
+    end
+    # Get a list of the pending instances
+    def list_of_pending_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /pending/}
+    end
+    # list of shutting down instances
+    def list_of_terminating_instances
+      list_of_nonterminated_instances.select {|a| a[:status] =~ /shutting/}
+    end
+    # list all the nonterminated instances
+    def list_of_nonterminated_instances
+      list_of_instances.reject {|a| a[:status] =~ /terminated/}
+    end
+    # List the instances, regardless of their states
+    def list_of_instances
+      get_instances_description
+    end
+    # Get number of pending instances
+    def number_of_pending_instances
+      list_of_pending_instances.size
+    end
+    # get the number of running instances
+    def number_of_running_instances
+      list_of_running_instances.size
+    end
+    # get the number of pending and running instances
+    def number_of_pending_and_running_instances
+      number_of_running_instances + number_of_pending_instances
+    end
+    # == LAUNCHING
+    # Request to launch a new instance
+    def request_launch_new_instance
+      if can_start_a_new_instance?
+        request_launch_one_instance_at_a_time
+        return true
+      else
+        return false
+      end
+    end
+    # Can we start a new instance?
+    def can_start_a_new_instance?
+      maximum_number_of_instances_are_not_running?
+    end
+    # Are the maximum number of instances running?
+    def maximum_number_of_instances_are_not_running?
+      list_of_running_instances.size < Application.maximum_instances
+    end
+    # Request to launch a number of instances
+    def request_launch_new_instances(num=1)
+      out = []
+      num.times {out << request_launch_one_instance_at_a_time}
+      out
+    end
+    # Launch one instance at a time
+    def request_launch_one_instance_at_a_time
+      reset!
+      while !number_of_pending_instances.zero?
+        wait "5.seconds"
+        reset!
+      end
+      return launch_new_instance!
+    end
+    # == SHUTDOWN
+    # Terminate all running instances
+    def request_termination_of_running_instances
+      list_of_running_instances.each {|a| terminate_instance!(a[:instance_id])}
+    end
+    # Request termination of all instances regardless of their state (includes pending instances)
+    def request_termination_of_all_instances
+      get_instances_description.each {|a| terminate_instance!(a[:instance_id])}
+    end
+    # Terminate instance by id
+    def request_termination_of_instance(id)
+      if can_shutdown_an_instance?
+        terminate_instance! id
+        return true
+      else
+        return false
+      end
+    end
+    # Can we shutdown an instance?
+    def can_shutdown_an_instance?
+      minimum_number_of_instances_are_running?
+    end
+    # Are the minimum number of instances running?
+    def minimum_number_of_instances_are_running?
+      list_of_running_instances.size > Application.minimum_instances
+    end
+    # Get the cached running_instances
+    def running_instances
+      @running_instances ||= update_instance_values
+    end
+    # Update the instance values
+    def update_instance_values
+      @running_instances = list_of_running_instances.collect {|a| RemoteInstance.new(a) }.sort
+    end
+  end
+    
+end

data/lib/poolparty/scheduler.rb

@@ -0,0 +1,103 @@
+module PoolParty
+  extend self
+  # Schedule tasks container
+  class ScheduleTasks
+    include ThreadSafeInstance
+    # Initialize tasks array and run
+    def tasks
+      @_tasks ||= []
+    end
+    # Synchronize the running threaded tasks
+    def run
+      unless tasks.empty?
+        self.class.synchronize do
+          tasks.reject!{|a|            
+            begin
+              a.run
+              a.join
+            rescue Exception => e    
+              puts "There was an error in the task: #{e} #{e.backtrace.join("\n")}"
+            end
+            true
+          }
+        end
+      end
+    end
+    # Add a task in a new thread
+    def <<(a, *args)
+      thread = Thread.new(a) do |task|
+        Thread.stop
+        Thread.current[:callee] = task
+        a.call args
+      end
+      tasks << thread
+    end
+    alias_method :push, :<<
+    # In the ThreadSafeInstance
+    make_safe :<<
+  end
+  # Scheduler class
+  module Scheduler
+    attr_reader :tasker
+    # Get the tasks or ScheduleTasks
+    def _tasker
+      @_tasker ||= ScheduleTasks.new
+    end
+    # Add a task to the new threaded block
+    def add_task(&blk)
+      _tasker.push proc{blk.call}
+    end
+    # Grab the polling_time
+    def interval
+      @interval ||= Application.polling_time
+    end
+    # Run the threads
+    def run_threads
+      _tasker.run
+    end
+    alias_method :run_tasks, :run_threads
+    # Daemonize the process
+    def daemonize
+      PoolParty.message "Daemonizing..."
+      
+      pid = fork do
+        Signal.trap('HUP', 'IGNORE') # Don't die upon logout
+        File.open("/dev/null", "r+") do |devnull|
+          $stdout.reopen(devnull)
+          $stderr.reopen(devnull)
+          $stdin.reopen(devnull) unless @use_stdin
+        end
+        yield if block_given?
+      end
+      Process.detach(pid)
+      pid
+    end
+    # Run the loop and wait the amount of time between running the tasks
+    # You can send it daemonize => true and it will daemonize
+    def run_thread_loop(opts={}, &block)
+      block = lambda {        
+        loop do
+          begin
+            run_thread_list(&block)
+            wait interval
+            reset!
+          rescue Exception => e
+            puts "There was an error in the run_thread_loop: #{e}"
+          end
+        end
+      }
+      # Run the tasks
+      opts[:daemonize] ? daemonize(&block) : block.call   
+    end
+    
+    def run_thread_list
+      yield if block_given?
+      run_threads
+    end
+    # Reset
+    def reset!
+      cached_variables.each {|cached| cached = nil }
+    end
+        
+  end
+end

data/lib/poolparty/tasks/cloud.rake

@@ -0,0 +1,57 @@
+# Cloud tasks
+namespace(:cloud) do
+  # Setup
+  task :init do
+    setup_application
+    raise Exception.new("You must specify your access_key and secret_access_key") unless Application.access_key && Application.secret_access_key
+  end
+  # Install the stack on all of the nodes
+  desc "Prepare all servers"
+  task :prepare => :init do
+    PoolParty::Master.new.nodes.each do |node|
+      node.install
+    end
+  end
+  # Start the cloud
+  desc "Start the cloud"
+  task :start => :init do
+    PoolParty::Master.new.start_cloud!
+  end
+  # Reload the cloud with the new updated data
+  desc "Reload all instances with updated data"
+  task :reload => :init do
+    PoolParty::Master.new.nodes.each do |node|
+      node.configure
+      node.restart_with_monit
+    end
+  end
+  # List the cloud
+  desc "List cloud"
+  task :list => :init do
+    PoolParty::Master.new.list
+  end
+  # Shutdown the cloud
+  desc "Shutdown the entire cloud"
+  task :shutdown => :init do
+    PoolParty::Master.new.request_termination_of_all_instances
+  end
+  # Watch the cloud and scale it if necessary
+  desc "Watch the cloud and maintain it"
+  task :scale => :init do
+    begin
+      PoolParty::Master.new.scale_cloud!
+    rescue Exception => e
+      puts "There was an error scaling the cloud: #{e}"
+    end
+    
+  end
+  # Maintain the cloud in a background process
+  desc "Maintain the cloud (run on the master)"
+  task :maintain => :init do
+    begin
+      PoolParty::Master.new.start_monitor!
+    rescue Exception => e
+      puts "There was an error starting the monitor: #{e}"
+    end
+  end
+end

data/lib/poolparty/tasks/development.rake

@@ -0,0 +1,38 @@
+namespace(:dev) do
+  task :init do
+    setup_application
+    run "mkdir ~/.ec2 >/dev/null 2>/dev/null" unless File.directory?("~/.ec2")      
+  end
+  # Setup a basic development environment for the user 
+  desc "Setup development environment specify the config_file"
+  task :setup => [:init] do
+    keyfilename = ".#{Application.keypair}_pool_keys"
+    run <<-EOR
+      echo 'export AWS_ACCESS_KEY_ID=\"#{Application.access_key}\"' > $HOME/#{keyfilename}
+      echo 'export AWS_SECRET_ACCESS_ID=\"#{Application.secret_access_key}\"' >> $HOME/#{keyfilename}
+      echo 'export EC2_HOME=\"#{Application.ec2_dir}\"' >> $HOME/#{keyfilename}
+      echo 'export KEYPAIR_NAME=\"#{Application.keypair}\"' >> $HOME/#{keyfilename}
+      echo 'export CONFIG_FILE=\"#{Application.config_file}\"' >> $HOME/#{keyfilename}
+      echo 'export EC2_PRIVATE_KEY=`ls ~/.ec2/#{Application.keypair}/pk-*.pem`;' >> $HOME/#{keyfilename}
+      echo 'export EC2_CERT=`ls ~/.ec2/#{Application.keypair}/cert-*.pem`;' >> $HOME/#{keyfilename}
+      source $HOME/#{keyfilename}
+    EOR
+  end
+  desc "Generate a keypair"
+  task :setup_keypair => :init do
+    unless File.file?(Application.keypair_path)
+      Application.keypair = "cloud"
+      run <<-EOR
+        ec2-add-keypair cloud > #{Application.keypair_path}
+        chmod 600 #{Application.keypair_path}
+      EOR
+    end
+  end
+  desc "Authorize base ports for application"
+  task :authorize_ports => :init do
+    run <<-EOR
+      ec2-authorize -p 22 default
+      ec2-authorize -p 80 default
+    EOR
+  end
+end

data/lib/poolparty/tasks/ec2.rake

@@ -0,0 +1,20 @@
+namespace(:ec2) do        
+  task :init do
+    Application.options
+  end
+  # Start a new instance in the cloud
+  desc "Add and start an instance to the pool"
+  task :start_new_instance => [:init] do
+    puts PoolParty::Remoting.new.launch_new_instance!
+  end
+  # Stop all the instances via command-line
+  desc "Stop all running instances"
+  task :stop_running_instances => [:init] do
+    Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-terminate-instances`}
+  end
+  # Reboot the instances via commandline
+  desc "Restart all running instances"
+  task :restart_running_instances => [:init] do
+    Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-reboot-instances`}
+  end
+end

data/lib/poolparty/tasks/instance.rake

@@ -0,0 +1,63 @@
+namespace(:instance) do
+  # Find the instance we want to deal with
+  # interface can be: num=0, i=0, inst=0, 0
+  # defaults to the master instance (0)
+  task :init do
+    num = (ENV['num'] || ENV["i"] || ENV["inst"] || ARGV.shift || 0).to_i
+    raise Exception.new("Please set the number of the instance (i.e. num=1, i=1, or as an argument)") unless num
+    @node = PoolParty::Master.new.get_node(num)
+  end
+  # Ssh into the node
+  desc "Remotely login to the remote instance"
+  task :ssh => [:init] do
+    @node.ssh
+  end
+  desc "Send a file to the remote instance"
+  task :exec => :init do
+    @node.ssh ENV['cmd']
+  end
+  # Send a file to the remote instance
+  # as designated by src='' and dest=''
+  desc "Send a file to the remote instance"
+  task :scp => :init do
+    @node.scp ENV['src'], ENV['dest']
+  end
+  # Execute a command on the remote instance as designated
+  # by cmd=''
+  desc "Execute cmd on a remote instance"
+  task :exec => [:init] do
+    cmd = ENV['cmd'] || "ls -l"
+    puts @node.ssh(cmd.runnable)
+  end
+  # Restart all the services monitored by monit
+  desc "Restart all the services"
+  task :reload => [:init] do
+    @node.restart_with_monit
+  end
+  # Start all the services monitored by monit
+  desc "Start all services"
+  task :load => [:init] do
+    @node.start_with_monit
+  end
+  # Stop the services monitored by monit
+  desc "Stop all services"
+  task :stop => [:init] do
+    @node.stop_with_monit
+  end
+  # Install the required services on this node
+  desc "Install stack on this node"
+  task :install => :init do          
+    @node.install
+  end
+  # Turnoff this instance
+  desc "Teardown instance"
+  task :shutdown => :init do
+    `ec2-terminate-instances #{@node.instance_id}`
+  end
+  # Configure this node and start the services
+  desc "Configure the stack on this node"
+  task :configure => :init do
+    @node.configure
+    @node.restart_with_monit
+  end
+end

data/lib/poolparty/tasks/plugins.rake

@@ -0,0 +1,30 @@
+namespace(:plugin) do
+  task :init do
+    @command = ARGV.shift # Get rid of the command
+    @name = (ENV['location'] || ENV["l"] || ARGV.shift)
+    unless @name
+      puts <<-EOM
+Usage:
+rake #{@command} location
+
+Example:
+rake #{@command} git://github.com/auser/pool-party-plugins.git
+
+Check the help does for more information how to install a plugin
+http://poolpartyrb.com
+        
+      EOM
+      exit 
+    end    
+  end
+  desc "Install a plugin from a git repository"
+  task :install => :init do |command|
+    PoolParty::PluginManager.install_plugin @name
+  end
+  desc "Remove an installed plugin"
+  task :remove => :init do |command|
+    PoolParty::PluginManager.remove_plugin @name
+  end
+  rule "" do |t|
+  end
+end

data/lib/poolparty/tasks/server.rake

@@ -0,0 +1,42 @@
+# Tasks to be run on the server
+namespace(:server) do                
+  task :init  do
+    PoolParty::Coordinator.init(false)
+  end
+  # bundle, upload and register your bundle on the server
+  desc "Bundle, upload and register your ami"
+  task :all => [:bundle, :upload, :register] do
+    puts "== your ami is ready"
+  end
+  # Cleanup the /mnt directory
+  desc "Clean the /mnt directory"
+  task :clean_mnt do
+    `rm -rf /mnt/image* img*`
+  end
+  # Before we can bundle, we have to make sure we have the cert and pk files
+  desc "Ensure the required bundle files are present in /mnt"
+  task :check_bundle_files do
+    raise Exception.new("You must have a private key in your /mnt directory") unless File.exists?("/mnt/pk-*.pem")
+    raise Exception.new("You must have your access key in your /mnt directory") unless File.exists?("/mnt/cert-*.pem")          
+  end
+  # Bundle the image
+  desc "Bundle this image into the /mnt directory"
+  task :bundle => [:clean_mnt, :check_bundle_files] do
+    puts `ec2-bundle-vol -k /mnt/pk-*.pem -u '#{Planner.user_id}' -d /mnt -c /mnt/cert-*.pem -r i386`
+  end
+  # Upload the bundle into the app_name bucket
+  desc "Upload the bundle to your bucket with a unique name: deletes old ami"
+  task :upload => [:init, :delete_bucket] do
+    puts `ec2-upload-bundle -b #{Planner.app_name} -m /mnt/image.manifest.xml -a #{Planner.access_key} -s #{Planner.secret_access_key}`
+  end
+  # Register the bucket with amazon and get back an ami
+  desc "Register the bundle with amazon"
+  task :register do
+    puts `ec2-register -K /mnt/pk-*.pem -C /mnt/cert-*.pem #{Planner.app_name}/image.manifest.xml`
+  end
+  # Delete the bucket
+  desc "Delete the bucket with the bundle under tha app name"
+  task :delete_bucket do
+    Planner.app_name.delete_bucket 
+  end
+end

data/lib/poolparty/tasks.rb

@@ -0,0 +1,29 @@
+module PoolParty
+  class Tasks
+    include Callbacks
+    
+    # Setup and define all the tasks
+    def initialize
+      yield self if block_given?
+    end
+    # Define the tasks in the rakefile
+    # From the rakefile
+    def define_tasks
+      # Run the command on the local system
+      def run(cmd)
+        system(cmd.runnable)
+      end
+      # Basic setup action
+      def setup_application
+        PoolParty.options({:config_file => (ENV["CONFIG_FILE"] || ENV["config"]) })
+      end
+      
+      # Require the poolparty specific tasks
+      Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].each { |t| eval open(t).read }
+      
+      Dir["#{PoolParty.plugin_dir}/*/Rakefile"].each {|t| load "#{t}" }
+      
+      true
+    end    
+  end
+end

data/lib/poolparty/tmp.rb

@@ -0,0 +1,46 @@
+$:.unshift(File.dirname(__FILE__))
+require "backcall"
+require "remoter"
+class Test
+  include PoolParty::Remoter
+  include Callbacks
+  
+  after :initialize, :set_hosts
+  def rt
+    @rt ||= Rake::RemoteTask
+  end
+  
+  def set_hosts(c)
+    rt.host "myslice", :app, :db
+  end
+  
+  def rtask(name, *args, &block)
+    rt.remote_task(name.to_sym => args, &block)
+  end
+    
+  def scp local, remote
+    require "tempfile"
+    rtask(:scp) do
+      put remote do
+        open(local).read
+      end
+    end.execute
+  end
+  before :scp, :set_hosts
+  def ssh command=nil, &block
+    block = Proc.new do
+      run command
+    end
+    rtask(:ssh, &block).execute
+  end
+  before :ssh, :set_hosts
+end
+
+t = Test.new
+t.scp("/Users/auser/Sites/work/citrusbyte/internal/gems/pool-party/pool/CHANGELOG", "ho")
+t.ssh("ls -l")
+t.ssh <<-EOE
+  ls -l
+  mv ho CHANGELOG
+  cat CHANGELOG
+EOE

data/lib/poolparty.rb

@@ -0,0 +1,105 @@
+=begin rdoc
+  The main file, contains the client and the server application methods
+=end
+$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
+
+# rubygems
+require 'rubygems'
+require "aws/s3"
+require "sqs"
+require "EC2"
+require 'thread'
+require "pp"
+require "tempfile"
+require "aska"
+require "vlad"
+begin
+  require 'fastthread'
+  require 'thin'
+  require 'system_timer'
+  Timer = SystemTimer  
+rescue LoadError
+  require 'timeout'
+  Timer = Timeout
+end
+
+## Load PoolParty
+pwd = File.dirname(__FILE__)
+
+# Load the required files
+# If there is an init file, load that, otherwise
+# require all the files in each directory
+%w(core modules s3 poolparty).each do |dir|  
+  Dir["#{pwd}/#{dir}"].each do |dir|
+    begin
+      require File.join(dir, "init")
+    rescue LoadError => e
+      Dir["#{pwd}/#{File.basename(dir)}/**"].each {|file| require File.join(dir, File.basename(file))}
+    end
+  end
+end
+
+module PoolParty
+  module Version #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 8
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+  # PoolParty options
+  def options(opts={})
+    Application.options(opts)
+  end
+  # Are we working in verbose-mode
+  def verbose?
+    options.verbose == true
+  end
+  # Send a message if we are in verbose-mode
+  def message(msg="")
+    puts "-- #{msg}" if verbose?
+  end
+  # Root directory of the application
+  def root_dir
+    File.expand_path(File.dirname(__FILE__) + "/..")
+  end
+  # User directory
+  def user_dir
+    Dir.pwd
+  end
+  # Write string to a tempfile
+  def write_to_temp_file(str="")
+    tempfile = Tempfile.new("rand#{rand(1000)}-#{rand(1000)}")
+    tempfile.print(str)
+    tempfile.flush
+    tempfile
+  end
+  def register_monitor(*names)
+    names.each do |name|
+      PoolParty::Monitors.extend name
+      
+      PoolParty::Master.send :include, name::Master
+      PoolParty::RemoteInstance.send :include, name::Remote
+    end
+  end
+  def load_plugins
+    Dir["#{plugin_dir}/**/init.rb"].each {|a| require a}
+  end
+  def reset!
+    @@installed_plugins = nil
+    Application.options = nil
+  end
+  def plugin_dir
+    "#{user_dir}/vendor"
+  end
+  def read_config_file(filename)
+    return {} unless filename
+    YAML.load(open(filename).read)
+  end
+  def include_cloud_tasks
+    Tasks.new.define_tasks
+  end
+  
+  alias_method :tasks, :include_cloud_tasks
+  alias_method :include_tasks, :include_cloud_tasks
+end

data/lib/s3/s3_object_store_folders.rb

@@ -0,0 +1,44 @@
+=begin rdoc
+  S3 overloads
+=end
+module AWS
+  module S3
+    class S3Object
+       class << self
+         
+        alias :original_store :store
+        def store(key, data, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          original_store(key, data, bucket, options)
+        end
+        
+        def streamed_store(key, filepath, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          store(key,File.open(filepath), bucket)
+        end
+        
+        def store_directory(directory, bucket, options = {})
+          Dir[File.join(directory, "*")].each do |file|
+            streamed_store("#{File.basename(File.dirname(file))}/#{File.basename(file)}", file, bucket, options.update(:use_virtual_directories => true))
+          end
+        end
+  
+        def store_folders(key, bucket = nil, options = {})
+          folders = key.split("/")
+          folders.slice!(0)
+          folders.pop
+          current_folder = "/"
+          folders.each {|folder|
+            current_folder += folder
+            store_folder(current_folder, bucket, options)
+            current_folder += "/"
+          }
+        end
+  
+        def store_folder(key, bucket = nil, options = {})
+          original_store(key + "_$folder$", "", bucket, options) # store the magic entry that emulates a folder
+        end
+      end
+    end
+  end
+end