dreadpiratepj-poolparty 0.0.8

data/lib/poolparty/tasks/ec2.rake

@@ -0,0 +1,20 @@
+namespace(:ec2) do        
+  task :init do
+    Application.options
+  end
+  # Start a new instance in the cloud
+  desc "Add and start an instance to the pool"
+  task :start_new_instance => [:init] do
+    puts PoolParty::Remoting.new.launch_new_instance!
+  end
+  # Stop all the instances via command-line
+  desc "Stop all running instances"
+  task :stop_running_instances => [:init] do
+    Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-terminate-instances`}
+  end
+  # Reboot the instances via commandline
+  desc "Restart all running instances"
+  task :restart_running_instances => [:init] do
+    Thread.new {`ec2-describe-instances | grep INSTANCE | grep running | awk '{print $2}' | xargs ec2-reboot-instances`}
+  end
+end

data/lib/poolparty/tasks/instance.rake

@@ -0,0 +1,63 @@
+namespace(:instance) do
+  # Find the instance we want to deal with
+  # interface can be: num=0, i=0, inst=0, 0
+  # defaults to the master instance (0)
+  task :init do
+    num = (ENV['num'] || ENV["i"] || ENV["inst"] || ARGV.shift || 0).to_i
+    raise Exception.new("Please set the number of the instance (i.e. num=1, i=1, or as an argument)") unless num
+    @node = PoolParty::Master.new.get_node(num)
+  end
+  # Ssh into the node
+  desc "Remotely login to the remote instance"
+  task :ssh => [:init] do
+    @node.ssh
+  end
+  desc "Send a file to the remote instance"
+  task :exec => :init do
+    @node.ssh ENV['cmd']
+  end
+  # Send a file to the remote instance
+  # as designated by src='' and dest=''
+  desc "Send a file to the remote instance"
+  task :scp => :init do
+    @node.scp ENV['src'], ENV['dest']
+  end
+  # Execute a command on the remote instance as designated
+  # by cmd=''
+  desc "Execute cmd on a remote instance"
+  task :exec => [:init] do
+    cmd = ENV['cmd'] || "ls -l"
+    puts @node.ssh(cmd.runnable)
+  end
+  # Restart all the services monitored by monit
+  desc "Restart all the services"
+  task :reload => [:init] do
+    @node.restart_with_monit
+  end
+  # Start all the services monitored by monit
+  desc "Start all services"
+  task :load => [:init] do
+    @node.start_with_monit
+  end
+  # Stop the services monitored by monit
+  desc "Stop all services"
+  task :stop => [:init] do
+    @node.stop_with_monit
+  end
+  # Install the required services on this node
+  desc "Install stack on this node"
+  task :install => :init do          
+    @node.install
+  end
+  # Turnoff this instance
+  desc "Teardown instance"
+  task :shutdown => :init do
+    `ec2-terminate-instances #{@node.instance_id}`
+  end
+  # Configure this node and start the services
+  desc "Configure the stack on this node"
+  task :configure => :init do
+    @node.configure
+    @node.restart_with_monit
+  end
+end

data/lib/poolparty/tasks/plugins.rake

@@ -0,0 +1,30 @@
+namespace(:plugin) do
+  task :init do
+    @command = ARGV.shift # Get rid of the command
+    @name = (ENV['location'] || ENV["l"] || ARGV.shift)
+    unless @name
+      puts <<-EOM
+Usage:
+rake #{@command} location
+
+Example:
+rake #{@command} git://github.com/auser/pool-party-plugins.git
+
+Check the help does for more information how to install a plugin
+http://poolpartyrb.com
+        
+      EOM
+      exit 
+    end    
+  end
+  desc "Install a plugin from a git repository"
+  task :install => :init do |command|
+    PoolParty::PluginManager.install_plugin @name
+  end
+  desc "Remove an installed plugin"
+  task :remove => :init do |command|
+    PoolParty::PluginManager.remove_plugin @name
+  end
+  rule "" do |t|
+  end
+end

data/lib/poolparty/tasks/server.rake

@@ -0,0 +1,42 @@
+# Tasks to be run on the server
+namespace(:server) do                
+  task :init  do
+    PoolParty::Coordinator.init(false)
+  end
+  # bundle, upload and register your bundle on the server
+  desc "Bundle, upload and register your ami"
+  task :all => [:bundle, :upload, :register] do
+    puts "== your ami is ready"
+  end
+  # Cleanup the /mnt directory
+  desc "Clean the /mnt directory"
+  task :clean_mnt do
+    `rm -rf /mnt/image* img*`
+  end
+  # Before we can bundle, we have to make sure we have the cert and pk files
+  desc "Ensure the required bundle files are present in /mnt"
+  task :check_bundle_files do
+    raise Exception.new("You must have a private key in your /mnt directory") unless File.exists?("/mnt/pk-*.pem")
+    raise Exception.new("You must have your access key in your /mnt directory") unless File.exists?("/mnt/cert-*.pem")          
+  end
+  # Bundle the image
+  desc "Bundle this image into the /mnt directory"
+  task :bundle => [:clean_mnt, :check_bundle_files] do
+    puts `ec2-bundle-vol -k /mnt/pk-*.pem -u '#{Planner.user_id}' -d /mnt -c /mnt/cert-*.pem -r i386`
+  end
+  # Upload the bundle into the app_name bucket
+  desc "Upload the bundle to your bucket with a unique name: deletes old ami"
+  task :upload => [:init, :delete_bucket] do
+    puts `ec2-upload-bundle -b #{Planner.app_name} -m /mnt/image.manifest.xml -a #{Planner.access_key} -s #{Planner.secret_access_key}`
+  end
+  # Register the bucket with amazon and get back an ami
+  desc "Register the bundle with amazon"
+  task :register do
+    puts `ec2-register -K /mnt/pk-*.pem -C /mnt/cert-*.pem #{Planner.app_name}/image.manifest.xml`
+  end
+  # Delete the bucket
+  desc "Delete the bucket with the bundle under tha app name"
+  task :delete_bucket do
+    Planner.app_name.delete_bucket 
+  end
+end

data/lib/poolparty/tmp.rb

@@ -0,0 +1,46 @@
+$:.unshift(File.dirname(__FILE__))
+require "backcall"
+require "remoter"
+class Test
+  include PoolParty::Remoter
+  include Callbacks
+  
+  after :initialize, :set_hosts
+  def rt
+    @rt ||= Rake::RemoteTask
+  end
+  
+  def set_hosts(c)
+    rt.host "myslice", :app, :db
+  end
+  
+  def rtask(name, *args, &block)
+    rt.remote_task(name.to_sym => args, &block)
+  end
+    
+  def scp local, remote
+    require "tempfile"
+    rtask(:scp) do
+      put remote do
+        open(local).read
+      end
+    end.execute
+  end
+  before :scp, :set_hosts
+  def ssh command=nil, &block
+    block = Proc.new do
+      run command
+    end
+    rtask(:ssh, &block).execute
+  end
+  before :ssh, :set_hosts
+end
+
+t = Test.new
+t.scp("/Users/auser/Sites/work/citrusbyte/internal/gems/pool-party/pool/CHANGELOG", "ho")
+t.ssh("ls -l")
+t.ssh <<-EOE
+  ls -l
+  mv ho CHANGELOG
+  cat CHANGELOG
+EOE

data/lib/s3/s3_object_store_folders.rb

@@ -0,0 +1,44 @@
+=begin rdoc
+  S3 overloads
+=end
+module AWS
+  module S3
+    class S3Object
+       class << self
+         
+        alias :original_store :store
+        def store(key, data, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          original_store(key, data, bucket, options)
+        end
+        
+        def streamed_store(key, filepath, bucket = nil, options = {})
+          store_folders(key, bucket, options) if options[:use_virtual_directories]
+          store(key,File.open(filepath), bucket)
+        end
+        
+        def store_directory(directory, bucket, options = {})
+          Dir[File.join(directory, "*")].each do |file|
+            streamed_store("#{File.basename(File.dirname(file))}/#{File.basename(file)}", file, bucket, options.update(:use_virtual_directories => true))
+          end
+        end
+  
+        def store_folders(key, bucket = nil, options = {})
+          folders = key.split("/")
+          folders.slice!(0)
+          folders.pop
+          current_folder = "/"
+          folders.each {|folder|
+            current_folder += folder
+            store_folder(current_folder, bucket, options)
+            current_folder += "/"
+          }
+        end
+  
+        def store_folder(key, bucket = nil, options = {})
+          original_store(key + "_$folder$", "", bucket, options) # store the magic entry that emulates a folder
+        end
+      end
+    end
+  end
+end

data/misc/basics_tutorial.txt

@@ -0,0 +1,142 @@
+PoolParty, a tutorial.
+
+To get PoolParty up and running on your very own ec2 instances, you have to do a few things in preparation first.
+
+First, install the gem with:
+
+sudo gem install auser-poolparty -s http://gems.github.com
+
+First, head on over to http://aws.amazon.com and sign up for an account. Make sure you make note of the access key and the secret access key on the access identifiers page. Also, scroll down to the bottom of that page and download your x.509 certificate and the private one as well. You only can generate that once, so make sure you download it immediately. Finally, take note of your account id.
+
+Now, let's get to some PoolParty setup. PoolParty only requires one file, a configuration file. Once you get this setup, PoolParty will do the rest for you, so let's get started editing the config file.
+
+There is a generator in progress, so if by the time you read this and you have the gem installed, type
+
+which poolparty
+
+If you get a response, then the generator has been built and you can skip this section and go right to the next 'using it' section, however, if not or you just want to understand the innerworkings, read on.
+
+config/config.yml
+PoolParty makes the guess that your config is in a file and is in config/config.yml in your base project directory. However, this is not necessary if you do not want a configuration file, since all options can be added at runtime using switches on the command-line. That being said, the easiest way is to write a config file.
+
+The minimal config file looks like this:
+
+:access_key: "1XCTNEK1CC5BQXXXXXXX"
+:secret_access_key: "Q2qJHP0S2iOKikn9glB+XXXX/XXXXXXXX/XXXXXXX"
+:ami: "ami-3057b259"
+
+That's it. However,a more complete one would look more like this:
+
+:app_name: "test_app"
+:user_id: "1619-6456-1111"
+:access_key: "1XCTNEK1CC5BQXXXXXXX"
+:secret_access_key: "Q2qJHP0S2iOKikn9glB+XXXX/XXXXXXXX/XXXXXXX"
+:ami: "ami-3057b259"
+:size: small
+:polling_time: "5.minutes"
+:minimum_instances: 2
+:maximum_instances: 10
+:ec2_dir: "/Users/user/.ec2"
+:keypair: user
+:os: ubuntu
+:host_port: 80
+:shared_bucket: "pool-party-app-data"
+:environment: production
+:contract_when:
+  web > 10
+  cpu < 0.2
+:expand_when:
+  cpu > 0.45
+  web < 10
+  
+Most of it should be self-explanatory, but a few points are necessary to describe. ec2_dir and keypair are for your location development machine. If you didn't create a keypair, not to worry, put in anything here. If you did, it's the name of the keypair you added. This is important for working with the instances after you get them up.
+
+If you put in a bucket in the shared_bucket key, the instances will automount that bucket to the /data drive. If you leave it out or don't include it, then it won't. 
+
+The master instance takes care of monitoring the cloud for you. If the the parameters you set in the contract_when and expand_when are met, then the cloud will respond accordingly. Those are based on monitors that are included in PoolParty.
+
+The installation of the required software for PoolParty is taken care of by PoolParty. That being said, PoolParty makes no assumptions of what the cloud's responsibilities will be. See 'Using' later on.
+
+Monitors
+The monitors are flexible and extensible. Currently included in PoolParty are monitors that monitor web requests and cpu and memory percentages. Writing a monitor or a plugin are out of the scope of this tutorial, but will be included in future tutorials.
+
+Configuring
+It can be tedious setting up just your development machine. This is why PoolParty makes it super easy for you in just one rake task. 
+
+If you are not familiar with ruby, I suggest you look into it, it's a beautiful language. However, PoolParty is language-agnostic, so not to worry.
+
+Create a file in your development directory called Rakefile and add this to it.
+
+require 'rubygems'
+require 'poolparty'
+
+PoolParty.include_tasks
+
+That's it, pretty simple and a bunch of tasks are available to you directly!
+Your development directory should look like this:
+
+./
+  config/
+    config.yml
+  Rakefile
+
+Now, once you have this setup and your config file setup, you can type
+
+rake dev:setup
+
+That's it and you should be able to type ec2-describe-images (provided you have installed the ec2-api-tools, available here: http://developer.amazonwebservices.com/connect/entry.jspa?externalID=351&categoryID=88) at this point and you are set.
+
+You will find yourself with a .<keypair_name>_pool_keys file in your home directory. Anytime you want to setup a new cloud or simply check on this one, all you have to do is type:
+
+source ~/.<keypair_name>_pool_keys 
+
+and your'll be ready to switch over to editing the new cloud. 
+
+Using:
+When it's easy to maintain a cloud, it's even easier to use.
+
+There are two binaries included with PoolParty when you install them. Working with the cloud, you can use the binary: pool. Working with the instance, you can use the binary: instance. 
+
+Start your pool by typing:
+
+pool start
+
+It may take a minute. PoolParty is starting your cloud and configuring the master at this point. You may be asked to confirm an ssh connection. This only happens the first time so other pool commands won't ask you again.
+
+To check on your cloud, simply type:
+
+pool list
+
+and you will be presented with the list of your cloud and their roles
+
+If you want to stop the pool, it's as straightforward as:
+
+pool stop
+
+There are a  other commands you have available to you on the pool site. If you are hosting your master off-site, rather than on EC2, you can type on the master machine:
+
+pool maintain
+
+and it will bootup as the master, watching the other instances in the cloud.
+
+If you want to log into a specific instance, you can do so with the command:
+
+instance ssh -i num
+
+The num is the number in the list from: pool list. It defaults to logging into the master if the number is not supplied.
+
+You have some more options when working with instances. You can 
+
+instance ssh
+instance scp src='' dest=''
+instance cmd cmd='ls -la'
+instance start|stop|install|restart
+instance start_maintain
+
+The last command will force the instance to become the new master.
+
+Those are the basics of PoolParty. 
+
+You can check it out at http://poolpartyrb.com/.
+
+If you are interested in active development, check out the source at github at http://github.com/auser/pool-party/tree/master. Active discussion is held here: http://groups.google.com/group/poolpartyrb. Of course, the latest and greatest information will always be released first at http://blog.citrusbyte.com.

data/poolparty.gemspec

@@ -0,0 +1,72 @@
+
+# Gem::Specification for Poolparty-0.0.8
+# Originally generated by Echoe
+
+Gem::Specification.new do |s|
+  s.name = %q{poolparty}
+  s.version = "0.0.8"
+
+  s.specification_version = 2 if s.respond_to? :specification_version=
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Ari Lerner"]
+  s.date = %q{2008-06-20}
+  s.description = %q{Run your entire application off EC2, managed and auto-scaling}
+  s.email = %q{ari.lerner@citrusbyte.com}
+  s.executables = ["instance", "pool"]
+  s.extra_rdoc_files = ["bin/instance", "bin/pool", "CHANGELOG", "lib/core/array.rb", "lib/core/exception.rb", "lib/core/float.rb", "lib/core/hash.rb", "lib/core/kernel.rb", "lib/core/module.rb", "lib/core/object.rb", "lib/core/proc.rb", "lib/core/string.rb", "lib/core/time.rb", "lib/modules/callback.rb", "lib/modules/ec2_wrapper.rb", "lib/modules/safe_instance.rb", "lib/modules/vlad_override.rb", "lib/poolparty/application.rb", "lib/poolparty/init.rb", "lib/poolparty/master.rb", "lib/poolparty/monitors/cpu.rb", "lib/poolparty/monitors/memory.rb", "lib/poolparty/monitors/web.rb", "lib/poolparty/monitors.rb", "lib/poolparty/optioner.rb", "lib/poolparty/plugin.rb", "lib/poolparty/plugin_manager.rb", "lib/poolparty/provider/packages/essential.rb", "lib/poolparty/provider/packages/git.rb", "lib/poolparty/provider/packages/haproxy.rb", "lib/poolparty/provider/packages/heartbeat.rb", "lib/poolparty/provider/packages/monit.rb", "lib/poolparty/provider/packages/rsync.rb", "lib/poolparty/provider/packages/ruby.rb", "lib/poolparty/provider/packages/s3fuse.rb", "lib/poolparty/provider/provider.rb", "lib/poolparty/provider.rb", "lib/poolparty/remote_instance.rb", "lib/poolparty/remoter.rb", "lib/poolparty/remoting.rb", "lib/poolparty/scheduler.rb", "lib/poolparty/tasks/cloud.rake", "lib/poolparty/tasks/development.rake", "lib/poolparty/tasks/ec2.rake", "lib/poolparty/tasks/instance.rake", "lib/poolparty/tasks/plugins.rake", "lib/poolparty/tasks/server.rake", "lib/poolparty/tasks.rb", "lib/poolparty/tmp.rb", "lib/poolparty.rb", "lib/s3/s3_object_store_folders.rb", "README.txt"]
+  s.files = ["archives/ruby-1.8.6-p111.tar.gz", "bin/instance", "bin/pool", "CHANGELOG", "config/cloud_master_takeover", "config/create_proxy_ami.sh", "config/haproxy.conf", "config/heartbeat.conf", "config/heartbeat_authkeys.conf", "config/installers/ubuntu_install.sh", "config/monit/haproxy.monit.conf", "config/monit/nginx.monit.conf", "config/monit.conf", "config/nginx.conf", "config/reconfigure_instances_script.sh", "config/sample-config.yml", "config/scp_instances_script.sh", "lib/core/array.rb", "lib/core/exception.rb", "lib/core/float.rb", "lib/core/hash.rb", "lib/core/kernel.rb", "lib/core/module.rb", "lib/core/object.rb", "lib/core/proc.rb", "lib/core/string.rb", "lib/core/time.rb", "lib/modules/callback.rb", "lib/modules/ec2_wrapper.rb", "lib/modules/safe_instance.rb", "lib/modules/vlad_override.rb", "lib/poolparty/application.rb", "lib/poolparty/init.rb", "lib/poolparty/master.rb", "lib/poolparty/monitors/cpu.rb", "lib/poolparty/monitors/memory.rb", "lib/poolparty/monitors/web.rb", "lib/poolparty/monitors.rb", "lib/poolparty/optioner.rb", "lib/poolparty/plugin.rb", "lib/poolparty/plugin_manager.rb", "lib/poolparty/provider/packages/essential.rb", "lib/poolparty/provider/packages/git.rb", "lib/poolparty/provider/packages/haproxy.rb", "lib/poolparty/provider/packages/heartbeat.rb", "lib/poolparty/provider/packages/monit.rb", "lib/poolparty/provider/packages/rsync.rb", "lib/poolparty/provider/packages/ruby.rb", "lib/poolparty/provider/packages/s3fuse.rb", "lib/poolparty/provider/provider.rb", "lib/poolparty/provider.rb", "lib/poolparty/remote_instance.rb", "lib/poolparty/remoter.rb", "lib/poolparty/remoting.rb", "lib/poolparty/scheduler.rb", "lib/poolparty/tasks/cloud.rake", "lib/poolparty/tasks/development.rake", "lib/poolparty/tasks/ec2.rake", "lib/poolparty/tasks/instance.rake", "lib/poolparty/tasks/plugins.rake", "lib/poolparty/tasks/server.rake", "lib/poolparty/tasks.rb", "lib/poolparty/tmp.rb", "lib/poolparty.rb", "lib/s3/s3_object_store_folders.rb", "Manifest", "misc/basics_tutorial.txt", "Rakefile", "README.txt", "spec/application_spec.rb", "spec/callback_spec.rb", "spec/core_spec.rb", "spec/helpers/ec2_mock.rb", "spec/kernel_spec.rb", "spec/master_spec.rb", "spec/monitors/cpu_monitor_spec.rb", "spec/monitors/memory_spec.rb", "spec/monitors/misc_monitor_spec.rb", "spec/monitors/web_spec.rb", "spec/optioner_spec.rb", "spec/plugin_manager_spec.rb", "spec/plugin_spec.rb", "spec/pool_binary_spec.rb", "spec/poolparty_spec.rb", "spec/provider_spec.rb", "spec/remote_instance_spec.rb", "spec/remoter_spec.rb", "spec/remoting_spec.rb", "spec/scheduler_spec.rb", "spec/spec_helper.rb", "spec/string_spec.rb", "tmp/node0-pool-party-ha.cf", "tmp/node0-pool-party-haresources", "tmp/node0-pool-party-hosts", "tmp/node1-pool-party-ha.cf", "tmp/node1-pool-party-haresources", "tmp/node1-pool-party-hosts", "tmp/pool-party-haproxy.cfg", "web/static/conf/nginx.conf", "web/static/site/images/balloon.png", "web/static/site/images/cb.png", "web/static/site/images/clouds.png", "web/static/site/images/railsconf_preso_img.png", "web/static/site/index.html", "web/static/site/javascripts/application.js", "web/static/site/javascripts/corner.js", "web/static/site/javascripts/jquery-1.2.6.pack.js", "web/static/site/misc.html", "web/static/site/storage/pool_party_presentation.pdf", "web/static/site/stylesheets/application.css", "web/static/site/stylesheets/reset.css", "web/static/src/layouts/application.haml", "web/static/src/pages/index.haml", "web/static/src/pages/misc.haml", "web/static/src/stylesheets/application.sass", "poolparty.gemspec"]
+  s.has_rdoc = true
+  s.homepage = %q{http://blog.citrusbyte.com}
+  s.post_install_message = %q{    Thanks for installing PoolParty!
+    
+    Please check out the documentation for any questions or check out the google groups at
+    http://groups.google.com/group/poolpartyrb
+    
+    Don't forget to check out the plugins for extending PoolParty!
+    
+    For more information, check http://poolpartyrb.com
+    *** Ari Lerner @ <ari.lerner@citrusbyte.com> ***
+}
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Poolparty", "--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{poolparty}
+  s.rubygems_version = %q{1.1.1}
+  s.summary = %q{Run your entire application off EC2, managed and auto-scaling}
+
+  s.add_dependency(%q<aws-s3>, [">= 0"])
+  s.add_dependency(%q<amazon-ec2>, [">= 0"])
+  s.add_dependency(%q<aska>, [">= 0"])
+  s.add_dependency(%q<git>, [">= 0"])
+end
+
+
+# # Original Rakefile source (requires the Echoe gem):
+# 
+# require 'rubygems'
+# require 'echoe'
+# require 'lib/poolparty'
+# 
+# task :default => :test
+# 
+# Echoe.new("poolparty") do |p|
+#   p.author = "Ari Lerner"
+#   p.email = "ari.lerner@citrusbyte.com"
+#   p.summary = "Run your entire application off EC2, managed and auto-scaling"
+#   p.url = "http://blog.citrusbyte.com"
+#   p.dependencies = %w(aws-s3 amazon-ec2 aska git)
+#   p.install_message =<<-EOM
+#     Thanks for installing PoolParty!
+#     
+#     Please check out the documentation for any questions or check out the google groups at
+#     http://groups.google.com/group/poolpartyrb
+#     
+#     Don't forget to check out the plugins for extending PoolParty!
+#     
+#     For more information, check http://poolpartyrb.com
+#     *** Ari Lerner @ <ari.lerner@citrusbyte.com> ***
+#   EOM
+#   p.include_rakefile = true
+# end
+# 
+# PoolParty.include_tasks