poolparty 0.0.4

data/lib/pool_party/tasks.rb

@@ -0,0 +1,220 @@
+module PoolParty
+  module TaskCommands
+    # Run the command on the local system
+    def run(cmd)
+      system(cmd.runnable)
+    end
+    # Basic setup action
+    def setup_application
+      Application.options({:config_file => (ENV["CONFIG_FILE"] || ENV["config"]) })
+    end
+  end
+  class Tasks
+    include TaskCommands
+    # Setup and define all the tasks
+    def initialize
+      yield self if block_given?
+      define_tasks!
+    end
+    # Define the tasks in the rakefile
+    def define_tasks!
+      # Tasks dealing with only an instance
+      namespace(:instance) do
+        # Find the instance we want to deal with
+        # interface can be: num=0, i=0, inst=0, 0
+        # defaults to the master instance (0)
+        task :init do
+          num = (ENV['num'] || ENV["i"] || ENV["inst"] || ARGV.shift || 0).to_i          
+          raise Exception.new("Please set the number of the instance (i.e. num=1, i=1, or as an argument)") unless num
+          @node = PoolParty::Master.new.get_node(num)
+        end
+        # Ssh into the node
+        desc "Remotely login to the remote instance"
+        task :ssh => [:init] do
+          @node.ssh
+        end
+        # Send a file to the remote instance
+        # as designated by src='' and dest=''
+        desc "Send a file to the remote instance"
+        task :scp => [:init] do
+          @node.scp ENV['src'], ENV['dest']
+        end
+        # Execute a command on the remote instance as designated
+        # by cmd=''
+        desc "Execute cmd on a remote instance"
+        task :exec => [:init] do
+          cmd = ENV['cmd'] || "ls -l"
+          puts @node.ssh(cmd.runnable)
+        end
+        # Restart all the services monitored by monit
+        desc "Restart all the services"
+        task :reload => [:init] do
+          @node.restart_with_monit
+        end
+        # Start all the services monitored by monit
+        desc "Start all services"
+        task :load => [:init] do
+          @node.start_with_monit
+        end
+        # Stop the services monitored by monit
+        desc "Stop all services"
+        task :stop => [:init] do
+          @node.stop_with_monit
+        end
+        # Install the required services on this node
+        desc "Install stack on this node"
+        task :install => :init do          
+          @node.install_stack
+        end
+        # Turnoff this instance
+        desc "Teardown instance"
+        task :shutdown => :init do
+          `ec2-terminate-instances #{@node.instance_id}`
+        end
+        # Configure this node and start the services
+        desc "Configure the stack on this node"
+        task :configure => :init do
+          @node.configure
+          @node.restart_with_monit
+        end
+      end
+      
+      namespace(:dev) do
+        task :init do
+          setup_application
+        end
+        # Setup a basic development environment for the user 
+        desc "Setup development environment specify the config_file"
+        task :setup => :init do
+          keyfilename = ".#{Application.keypair}_pool_keys"
+          run <<-EOR
+            echo 'export ACCESS_KEY=\"#{Application.access_key}\"' > $HOME/#{keyfilename}
+            echo 'export SECRET_ACCESS_KEY=\"#{Application.secret_access_key}\"' >> $HOME/#{keyfilename}
+            echo 'export EC2_HOME=\"#{Application.ec2_dir}\"' >> $HOME/#{keyfilename}
+            echo 'export KEYPAIR_NAME=\"#{Application.keypair}\"' >> $HOME/#{keyfilename}
+            echo 'export CONFIG_FILE=\"#{Application.config_file}\"' >> $HOME/#{keyfilename}
+          EOR
+        end
+      end
+      # Cloud tasks
+      namespace(:cloud) do
+        # Setup
+        task :init do
+          setup_application
+          raise Exception.new("You must specify your access_key and secret_access_key") unless Application.access_key && Application.secret_access_key
+        end
+        # Install the stack on all of the nodes
+        desc "Prepare all servers"
+        task :prepare => :init do
+          PoolParty::Master.new.nodes.each do |node|
+            node.install_stack
+          end
+        end
+        # Start the cloud
+        desc "Start the cloud"
+        task :start => :init do
+          PoolParty::Master.new.start_cloud!
+        end
+        # Reload the cloud with the new updated data
+        desc "Reload all instances with updated data"
+        task :reload => :init do
+          PoolParty::Master.new.nodes.each do |node|
+            node.configure
+            node.restart_with_monit
+          end
+        end
+        # List the cloud
+        desc "List cloud"
+        task :list => :init do
+          PoolParty::Master.new.list
+        end
+        # Shutdown the cloud
+        desc "Shutdown the entire cloud"
+        task :shutdown => :init do
+          PoolParty::Master.new.request_termination_of_all_instances
+        end
+        # Maintain the cloud in a background process
+        desc "Maintain the cloud (run on the master)"
+        task :maintain => :init do
+          begin
+            PoolParty::Master.new.start_monitor!
+          rescue Exception => e
+            puts "There was an error starting the monitor: #{e}"
+          end
+        end
+        # Deploy task. 
+        # TODO: Find a beautiful way of updating the user-defined configuration
+        # data
+        desc "Deploy web application from production git repos specified in config file"
+        task :deploy => :init do
+          puts "Deploying web app on nginx"
+        end
+      end
+      
+      # Nearly antiquated tasks
+      namespace(:ec2) do        
+        task :init do
+          Application.options
+        end
+        # Start a new instance in the cloud
+        desc "Add and start an instance to the pool"
+        task :start_new_instance => [:init] do
+          puts PoolParty::Remoting.new.launch_new_instance!
+        end
+        # Stop all the instances via command-line
+        desc "Stop all running instances"
+        task :stop_running_instances => [:init] do
+          Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-terminate-instances`}
+        end
+        # Reboot the instances via commandline
+        desc "Restart all running instances"
+        task :restart_running_instances => [:init] do
+          Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-reboot-instances`}
+        end
+      end
+      # Tasks to be run on the server
+      namespace(:server) do                
+        task :init  do
+          PoolParty::Coordinator.init(false)
+        end
+        # bundle, upload and register your bundle on the server
+        desc "Bundle, upload and register your ami"
+        task :all => [:bundle, :upload, :register] do
+          puts "== your ami is ready"
+        end
+        # Cleanup the /mnt directory
+        desc "Clean the /mnt directory"
+        task :clean_mnt do
+          `rm -rf /mnt/image* img*`
+        end
+        # Before we can bundle, we have to make sure we have the cert and pk files
+        desc "Ensure the required bundle files are present in /mnt"
+        task :check_bundle_files do
+          raise Exception.new("You must have a private key in your /mnt directory") unless File.exists?("/mnt/pk-*.pem")
+          raise Exception.new("You must have your access key in your /mnt directory") unless File.exists?("/mnt/cert-*.pem")          
+        end
+        # Bundle the image
+        desc "Bundle this image into the /mnt directory"
+        task :bundle => [:clean_mnt, :check_bundle_files] do
+          puts `ec2-bundle-vol -k /mnt/pk-*.pem -u '#{Planner.user_id}' -d /mnt -c /mnt/cert-*.pem -r i386`
+        end
+        # Upload the bundle into the app_name bucket
+        desc "Upload the bundle to your bucket with a unique name: deletes old ami"
+        task :upload => [:init, :delete_bucket] do
+          puts `ec2-upload-bundle -b #{Planner.app_name} -m /mnt/image.manifest.xml -a #{Planner.access_key} -s #{Planner.secret_access_key}`
+        end
+        # Register the bucket with amazon and get back an ami
+        desc "Register the bundle with amazon"
+        task :register do
+          puts `ec2-register -K /mnt/pk-*.pem -C /mnt/cert-*.pem #{Planner.app_name}/image.manifest.xml`
+        end
+        # Delete the bucket
+        desc "Delete the bucket with the bundle under tha app name"
+        task :delete_bucket do
+          Planner.app_name.delete_bucket 
+        end
+      end      
+    end    
+    
+  end
+end

data/lib/pool_party.rb

@@ -0,0 +1,69 @@
+=begin rdoc
+  The main file, contains the client and the server application methods
+=end
+$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
+
+# rubygems
+require 'rubygems'
+require "aws/s3"
+require "sqs"
+require "EC2"
+require 'thread'
+require "pp"
+require "tempfile"
+require "aska"
+begin
+  require 'fastthread'
+  require 'thin'
+rescue LoadError
+end
+
+## Load PoolParty
+pwd = File.dirname(__FILE__)
+
+# Load the required files
+# If there is an init file, load that, otherwise
+# require all the files in each directory
+%w(core modules s3 pool_party).each do |dir|  
+  Dir["#{pwd}/#{dir}"].each do |dir|
+    begin
+      require File.join(dir, "init")
+    rescue LoadError => e
+      Dir["#{pwd}/#{File.basename(dir)}/**"].each {|file| require File.join(dir, File.basename(file))}
+    end
+  end
+end
+
+module PoolParty  
+  module Version
+    MAJOR = '0'
+    MINOR = '0'
+    REVISION = '4'
+    def self.combined
+      [MAJOR, MINOR, REVISION].join('.')
+    end
+  end
+  # PoolParty options
+  def options(opts={})
+    Application.options(opts)
+  end
+  # Are we working in verbose-mode
+  def verbose?
+    Application.verbose == true
+  end
+  # Send a message if we are in verbose-mode
+  def message(msg="")
+    pp "-- #{msg}" if verbose?
+  end
+  # Root directory of the application
+  def root_dir
+    File.dirname(__FILE__)
+  end
+  # Write string to a tempfile
+  def write_to_temp_file(str="")
+    tempfile = Tempfile.new("rand#{rand(1000)}-#{rand(1000)}")
+    tempfile.print(str)
+    tempfile.flush
+    tempfile
+  end
+end

data/lib/s3/s3_object_store_folders.rb

@@ -0,0 +1,44 @@
+=begin rdoc
+  S3 overloads
+=end
+module AWS
+  module S3
+    class S3Object
+       class << self
+         
+        alias :original_store :store
+        def store(key, data, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          original_store(key, data, bucket, options)
+        end
+        
+        def streamed_store(key, filepath, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          store(key,File.open(filepath), bucket)
+        end
+        
+        def store_directory(directory, bucket, options = {})
+          Dir[File.join(directory, "*")].each do |file|
+            streamed_store("#{File.basename(File.dirname(file))}/#{File.basename(file)}", file, bucket, options.update(:use_virtual_directories => true))
+          end
+        end
+  
+        def store_folders(key, bucket = nil, options = {})
+          folders = key.split("/")
+          folders.slice!(0)
+          folders.pop
+          current_folder = "/"
+          folders.each {|folder|
+            current_folder += folder
+            store_folder(current_folder, bucket, options)
+            current_folder += "/"
+          }
+        end
+  
+        def store_folder(key, bucket = nil, options = {})
+          original_store(key + "_$folder$", "", bucket, options) # store the magic entry that emulates a folder
+        end
+      end
+    end
+  end
+end

data/poolparty.gemspec

@@ -0,0 +1,55 @@
+
+# Gem::Specification for Poolparty-0.0.4
+# Originally generated by Echoe
+
+Gem::Specification.new do |s|
+  s.name = %q{poolparty}
+  s.version = "0.0.4"
+
+  s.specification_version = 2 if s.respond_to? :specification_version=
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ari Lerner"]
+  s.date = %q{2008-05-28}
+  s.description = %q{Run your entire application off EC2, managed and auto-scaling}
+  s.email = %q{ari.lerner@citrusbyte.com}
+  s.executables = ["instance", "pool"]
+  s.extra_rdoc_files = ["bin/instance", "bin/pool", "CHANGELOG", "lib/core/array.rb", "lib/core/exception.rb", "lib/core/kernel.rb", "lib/core/module.rb", "lib/core/object.rb", "lib/core/string.rb", "lib/core/time.rb", "lib/modules/callback.rb", "lib/modules/ec2_wrapper.rb", "lib/modules/safe_instance.rb", "lib/pool_party/application.rb", "lib/pool_party/init.rb", "lib/pool_party/master.rb", "lib/pool_party/monitors/cpu.rb", "lib/pool_party/monitors/memory.rb", "lib/pool_party/monitors/web.rb", "lib/pool_party/monitors.rb", "lib/pool_party/optioner.rb", "lib/pool_party/os/ubuntu.rb", "lib/pool_party/os.rb", "lib/pool_party/remote_instance.rb", "lib/pool_party/remoting.rb", "lib/pool_party/scheduler.rb", "lib/pool_party/tasks.rb", "lib/pool_party.rb", "lib/s3/s3_object_store_folders.rb", "README.txt"]
+  s.files = ["bin/instance", "bin/pool", "CHANGELOG", "config/config.yml", "config/create_proxy_ami.sh", "config/haproxy.conf", "config/heartbeat.conf", "config/heartbeat_authkeys.conf", "config/monit/haproxy.monit.conf", "config/monit/nginx.monit.conf", "config/monit.conf", "config/nginx.conf", "lib/core/array.rb", "lib/core/exception.rb", "lib/core/kernel.rb", "lib/core/module.rb", "lib/core/object.rb", "lib/core/string.rb", "lib/core/time.rb", "lib/modules/callback.rb", "lib/modules/ec2_wrapper.rb", "lib/modules/safe_instance.rb", "lib/pool_party/application.rb", "lib/pool_party/init.rb", "lib/pool_party/master.rb", "lib/pool_party/monitors/cpu.rb", "lib/pool_party/monitors/memory.rb", "lib/pool_party/monitors/web.rb", "lib/pool_party/monitors.rb", "lib/pool_party/optioner.rb", "lib/pool_party/os/ubuntu.rb", "lib/pool_party/os.rb", "lib/pool_party/remote_instance.rb", "lib/pool_party/remoting.rb", "lib/pool_party/scheduler.rb", "lib/pool_party/tasks.rb", "lib/pool_party.rb", "lib/s3/s3_object_store_folders.rb", "Manifest", "Rakefile", "README.txt", "spec/application_spec.rb", "spec/callback_spec.rb", "spec/helpers/ec2_mock.rb", "spec/helpers/remote_instance_mock.rb", "spec/kernel_spec.rb", "spec/master_spec.rb", "spec/monitor_spec.rb", "spec/optioner_spec.rb", "spec/poolparty_spec.rb", "spec/remote_instance_spec.rb", "spec/remoting_spec.rb", "spec/spec_helper.rb", "spec/string_spec.rb", "test/test_pool_party.rb", "poolparty.gemspec"]
+  s.has_rdoc = true
+  s.homepage = %q{http://blog.citrusbyte.com}
+  s.post_install_message = %q{For more information, check http://poolpartyrb.com
+*** Ari Lerner @ <ari.lerner@citrusbyte.com> ***}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Poolparty", "--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{poolparty}
+  s.rubygems_version = %q{1.0.1}
+  s.summary = %q{Run your entire application off EC2, managed and auto-scaling}
+  s.test_files = ["test/test_pool_party.rb"]
+
+  s.add_dependency(%q<aws-s3>, [">= 0"])
+  s.add_dependency(%q<amazon-ec2>, [">= 0"])
+  s.add_dependency(%q<aska>, [">= 0"])
+end
+
+
+# # Original Rakefile source (requires the Echoe gem):
+# 
+# require 'rubygems'
+# require 'echoe'
+# require 'lib/pool_party'
+# 
+# task :default => :test
+# 
+# Echoe.new("poolparty") do |p|
+#   p.author = "Ari Lerner"
+#   p.email = "ari.lerner@citrusbyte.com"
+#   p.summary = "Run your entire application off EC2, managed and auto-scaling"
+#   p.url = "http://blog.citrusbyte.com"
+#   p.docs_host = "www.poolpartyrb.com"
+#   p.dependencies = %w(aws-s3 amazon-ec2 aska)
+#   p.install_message = "For more information, check http://poolpartyrb.com\n*** Ari Lerner @ <ari.lerner@citrusbyte.com> ***"
+#   p.include_rakefile = true
+# end
+# 
+# PoolParty::Tasks.new

data/spec/application_spec.rb

@@ -0,0 +1,32 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "Application" do
+  before(:each) do
+  end
+  it "should have the root_dir defined" do
+    Application.root_dir.should_not be_nil
+  end
+  it "should be able to call on the haproxy_config_file" do
+    Application.haproxy_config_file.should_not be_nil
+  end
+  it "should be able to find the client_port" do
+    Application.options.should_receive(:client_port).and_return(7788)
+    Application.client_port.should == 7788
+  end
+  it "should always have haproxy in the managed services list" do
+    Application.managed_services =~ /haproxy/
+  end
+  it "should be able to say it is in development mode if it is in dev mode" do
+    Application.stub!(:environment).and_return("development")
+    Application.development?.should == true
+  end
+  it "should be able to say it is in production if it is in production" do
+    Application.stub!(:environment).and_return("production")
+    Application.production?.should == true
+  end
+  it "should be able to say it's keypair path is in the $HOME/ directory" do
+    Application.stub!(:ec2_dir).and_return("~/.ec2")
+    Application.stub!(:keypair).and_return("poolparty")
+    Application.keypair_path.should == "~/.ec2/id_rsa-poolparty"
+  end
+end

data/spec/callback_spec.rb

@@ -0,0 +1,65 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+class TestCallbacks
+  include Callbacks
+  attr_reader :str
+  def hello
+    string << "hello "
+  end
+  def world
+    string << "world"
+  end
+  def thanks
+    string << ", thank you"
+  end
+  before :world, :hello
+  after :world, :thanks
+  def pop
+    string << "pop"
+  end
+  def boom
+    string << " goes boom"
+  end  
+  after :pop, :boom
+  def string
+    @str ||= String.new
+  end
+end
+describe "Callbacks" do
+  before(:each) do
+    @klass = TestCallbacks.new
+  end
+  it "should retain it's class identifier" do
+    @klass.class.should == TestCallbacks
+  end
+  it "should callback the method before the method runs" do
+    @klass.world.should == "hello world, thank you"
+  end
+  it "should callback the method before the method runs" do
+    @klass.pop.should == "pop goes boom"
+  end
+end
+class TestMultipleCallbacks
+  include Callbacks
+  attr_reader :str
+  def hi
+    string << "hi, "
+  end
+  def hello
+    string << "hello "
+  end
+  def world
+    string << "world"
+  end
+  def string
+    @str ||= String.new
+  end
+  before :world, :hi
+  before :world, :hello
+end
+describe "Multiple callbacks" do
+  before(:each) do
+    @klass = TestMultipleCallbacks.new
+  end
+  it "should be able to have multiple callbacks on the same call"
+end

data/spec/helpers/ec2_mock.rb

@@ -0,0 +1,56 @@
+module PoolParty
+  module EC2Mock
+    def launch_new_instance!
+      letter = ("a".."z").to_a[instances.size] # For unique instance_ids
+      h = {:instance_id => "i-58ba56#{letter}", :ip => "ip-127-0-0-1.aws.amazonaws.com", :status => "pending", :launching_time => Time.now }
+      instances << h      
+      Thread.new {wait 0.1;h[:status] = "running"} # Simulate the startup time
+      return h
+    end
+    # Shutdown the instance by instance_id
+    def terminate_instance!(instance_id)
+      instances.select {|a| a[:instance_id] == instance_id}[0][:status] = "terminating"
+    end
+    # Instance description
+    def describe_instance(id)
+      item = instances.select {|a| a[:instance_id] == id}[0]
+      EC2ResponseObject.get_hash_from_response(item)
+    end
+    # Get instance by id
+    def get_instance_by_id(id)
+      get_instances_description.select {|a| a.instance_id == id}[0] rescue nil
+    end
+    # Get the s3 description for the response in a hash format
+    def get_instances_description
+      instances
+    end
+    # Fake the ec2 connection
+    def ec2
+      @ec2 ||= EC2::Base.new(:access_key_id => "not a key", :secret_access_key => "not a key")
+    end
+    # Some basic instances, not totally necessary
+    def instances
+      @instances ||= []
+    end
+  end
+  class EC2ResponseObject
+    def self.get_descriptions(resp)
+      rs = resp.DescribeInstancesResponse.reservationSet.item
+      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      out = begin
+        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
+      rescue Exception => e
+        begin
+          # Really weird bug with amazon's ec2 gem
+          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
+        rescue Exception => e
+          []
+        end                
+      end
+      out
+    end
+    def self.get_hash_from_response(resp)
+      {:instance_id => resp.instanceId, :ip => resp.dnsName, :status => resp.instanceState.name, :launching_time => resp.launchTime} rescue nil
+    end
+  end
+end

data/spec/helpers/remote_instance_mock.rb

@@ -0,0 +1,11 @@
+module PoolParty
+  module RemoteInstanceMock
+    def scp(src="", dest="")
+      true
+    end
+    # Ssh into the instance or run a command, if the cmd is set
+    def ssh(cmd="")
+      true
+    end
+  end
+end

data/spec/kernel_spec.rb

@@ -0,0 +1,11 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe "Kernel extensions" do
+  before(:each) do
+    @host = Master.new
+  end
+  it "should eval the string into time" do
+    @host.should_receive(:sleep).once.and_return true
+    @host.wait "10.seconds"
+  end
+end