poolparty 0.0.4

data/lib/core/kernel.rb

@@ -0,0 +1,9 @@
+=begin rdoc
+  Kernel overloads
+=end
+module Kernel
+  # Nice wait instead of sleep
+  def wait(time=10)
+    sleep time.is_a?(String) ? eval(time) : time
+  end
+end

data/lib/core/module.rb

@@ -0,0 +1,22 @@
+# Module overloads
+class Module
+  # Gives us alias_method_chain from rails
+  def alias_method_chain(target, feature)
+    aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
+    yield(aliased_target, punctuation) if block_given?
+
+    with_method, without_method = "#{aliased_target}_with_#{feature}#{punctuation}", "#{aliased_target}_without_#{feature}#{punctuation}"
+
+    alias_method without_method, target
+    alias_method target, with_method
+
+    case
+      when public_method_defined?(without_method)
+        public target
+      when protected_method_defined?(without_method)
+        protected target
+      when private_method_defined?(without_method)
+        private target
+    end
+  end
+end

data/lib/core/object.rb

@@ -0,0 +1,14 @@
+=begin rdoc
+  Basic, add an alias_method to the object class
+  Add returning to the object
+=end
+class Object
+  def alias_method(new_id, original_id)
+    original = self.method(original_id).to_proc
+    define_method(new_id){|*args| original.call(*args)}
+  end
+  def returning(receiver)
+    yield receiver
+    receiver
+  end
+end

data/lib/core/string.rb

@@ -0,0 +1,49 @@
+class String
+  def hasherize(format=[])
+    hash = {}
+    i = 0
+    self.split(%r{[\n|\t|\s| ]+}).collect {|a| a.strip}.each do |f|
+      break unless format[i]
+      unless f == "" || f.nil?
+        hash[format[i]] = f
+        i+=1
+      end      
+    end
+    hash
+  end
+  def ^(h={})
+    self.gsub(/:([\w]+)/) {h[$1.to_sym] if h.include?($1.to_sym)}
+  end
+  def runnable
+    self.strip.gsub(/\n/, " && ")
+  end
+  def classify
+    self.capitalize
+  end
+  def bucket_objects
+    AWS::S3::Bucket.objects(self)
+  end
+  def bucket_object(key)
+    AWS::S3::S3Object.value key, self if bucket_object_exists?(key)
+  end
+  def bucket_object_exists?(key)
+    AWS::S3::S3Object.exists? key, self
+  end
+  def store_bucket_value(key, data)
+    AWS::S3::S3Object.store key, data, self unless bucket_object_exists?(key)
+  end
+  def delete_bucket_value(key)
+    AWS::S3::S3Object.delete(key, self) if bucket_object_exists?(key)
+  end
+  def bucket_exists?
+    begin
+      AWS::S3::Bucket.find(self)
+      return true
+    rescue
+      return false
+    end        
+  end
+  def delete_bucket
+    AWS::S3::Bucket.delete(self, :force => true) if bucket_exists?
+  end
+end

data/lib/core/time.rb

@@ -0,0 +1,41 @@
+=begin rdoc
+  Based off the rails Numeric class.
+  Gives us the ability to use nice phrases such as
+  30.seconds, 5.days, etc.
+=end
+class Numeric
+  def ago(time = Time.now)
+    time - self
+  end
+  alias :until :ago
+
+  def since(time = Time.now)
+    time + self
+  end
+  alias :from_now :since
+  
+  def seconds
+    self
+  end
+  alias :second :seconds
+
+  def minutes
+    self * 60
+  end
+  alias :minute :minutes  
+
+  def hours
+    self * 60.minutes
+  end
+  alias :hour :hours
+
+  def days
+    self * 24.hours
+  end
+  alias :day :days
+
+  def weeks
+    self * 7.days
+  end
+  alias :week :weeks
+end

data/lib/modules/callback.rb

@@ -0,0 +1,55 @@
+module PoolParty
+  module Callbacks
+    module ClassMethods
+      def callback(type,m,e,*args, &block)
+        
+        # Save the old method
+        if method_defined?("#{type}_#{m}".to_sym)
+          puts "method already defined"
+        end
+                
+        case type
+        when :before          
+          str=<<-EOD
+            def #{type}_#{m}
+              #{e}              
+              yield if block_given?
+              super
+            end
+          EOD
+        when :after
+          str=<<-EOD
+            def #{type}_#{m}
+              super
+              yield if block_given?
+              #{e}
+            end
+          EOD
+        end
+                
+        mMod = Module.new {eval str}
+        
+        module_eval %{alias_method :#{type}_#{m}, :#{m}}
+        
+        self.send :define_method, "#{m}".to_sym, Proc.new {
+          extend(mMod)
+          method("#{type}_#{m}".to_sym).call
+        }
+      end
+      def before(m,e,*args, &block)
+        callback(:before,m,e,*args, &block)
+      end
+      def after(m,e,*args, &block)
+        callback(:after,m,e,*args,&block)
+      end
+    end
+    
+    module InstanceMethods            
+    end
+    
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end

data/lib/modules/ec2_wrapper.rb

@@ -0,0 +1,74 @@
+module PoolParty
+  extend self
+  
+  module Ec2Wrapper
+    
+    module ClassMethods      
+    end
+    
+    module InstanceMethods
+      # Run a new instance, with the user_data and the ami described in the config
+      def launch_new_instance!
+        instance = ec2.run_instances(
+          :image_id => Application.ami, 
+          :user_data => "#{Application.launching_user_data}",
+          :minCount => 1,
+          :maxCount => 1,
+          :key_name => Application.keypair,
+          :size => "#{Application.size}")
+          
+        item = instance.RunInstancesResponse.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Shutdown the instance by instance_id
+      def terminate_instance!(instance_id)
+        ec2.terminate_instances(:instance_id => instance_id)
+      end
+      # Instance description
+      def describe_instance(id)
+        instance = ec2.describe_instances(:instance_id => id)
+        item = instance.DescribeInstancesResponse.reservationSet.item.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Get instance by id
+      def get_instance_by_id(id)
+        get_instances_description.select {|a| a.instance_id == id}[0] rescue nil
+      end
+      # Get the s3 description for the response in a hash format
+      def get_instances_description
+        @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
+      end
+            
+      # EC2 connections
+      def ec2
+        @ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
+      end      
+    end
+    
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+  # Provides a simple class to wrap around the amazon responses
+  class EC2ResponseObject
+    def self.get_descriptions(resp)
+      rs = resp.DescribeInstancesResponse.reservationSet.item
+      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      out = begin
+        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
+      rescue Exception => e
+        begin
+          # Really weird bug with amazon's ec2 gem
+          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
+        rescue Exception => e
+          []
+        end                
+      end
+      out
+    end
+    def self.get_hash_from_response(resp)
+      {:instance_id => resp.instanceId, :ip => resp.dnsName, :status => resp.instanceState.name, :launching_time => resp.launchTime} rescue nil
+    end
+  end
+end

data/lib/modules/safe_instance.rb

@@ -0,0 +1,31 @@
+=begin rdoc
+  Make a command thread-safe
+=end
+require "monitor"
+module PoolParty
+  extend self
+  
+  module ThreadSafeInstance
+    
+    module ClassMethods
+      def make_safe(meth)
+       original_method = "_unsafe_#{meth}_"
+       alias_method original_method, meth
+       define_method(meth) {|*args| self.class.synchronize { self.send(original_method) } }
+       self
+      end  
+    end
+    
+    module InstanceMethods
+      def make_safe meth
+        self.class.make_safe meth
+      end
+    end
+    
+    def self.included(receiver)
+      receiver.extend         MonitorMixin
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end

data/lib/pool_party/application.rb

@@ -0,0 +1,133 @@
+=begin rdoc
+  Application
+  This handles user interaction
+=end
+$:.unshift File.dirname(__FILE__)
+
+module PoolParty
+  extend self
+  
+  class Application
+    class << self
+      
+      # The application options
+      def options(opts={})
+        @options ||= make_options(opts)
+      end      
+      # Make the options with the config_file overrides included
+      # Default config file assumed to be at config/config.yml
+      def make_options(opts={})
+        load_options!
+        default_options.merge!(opts)
+        # If the config_file options are specified and not empty
+        unless default_options[:config_file].nil? || default_options[:config_file].empty?
+          require "yaml"
+          # Try loading the file if it exists
+          filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
+          default_options.merge!( YAML.load(filedata) ) if filedata
+        end
+
+        OpenStruct.new(default_options)
+      end
+
+      # Load options via commandline
+      def load_options!
+        require 'optparse'
+        OptionParser.new do |op|
+          op.on('-A key', '--access-key key', "Ec2 access key (ENV['ACCESS_KEY'])") { |key| default_options[:access_key] = key }
+          op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['SECRET_ACCESS_KEY'])") { |key| default_options[:secret_access_key] = key }
+          op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-4a46a323')") {|id| default_options[:ami] = id }
+          op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
+          op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }          
+          op.on('-D ec2 directory', '--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
+          op.on('-S services', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
+          op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
+          op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
+          op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }          
+          op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
+          op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }          
+          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
+          op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
+          op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
+          op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
+          op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
+          op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = v}
+          op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i}
+          op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x}
+
+          op.on_tail("-h", "--help", "Show this message") do |o|
+            puts "op: #{o}"
+            exit
+          end          
+        end.parse!(ARGV.dup)
+      end
+      
+      # Basic default options
+      # All can be overridden by the command line
+      # or in a config.yml file
+      def default_options
+        @default_options ||= {
+          :host_port => 80,
+          :client_port => 8001,
+          :environment => 'development',
+          :verbose => true,
+          :logging => true,
+          :size => "small",
+          :polling_time => "30.seconds",
+          :user_data => "",
+          :heavy_load => 0.80,
+          :light_load => 0.15,
+          :minimum_instances => 1,
+          :maximum_instances => 3,
+          :access_key => ENV["ACCESS_KEY"],
+          :secret_access_key => ENV["SECRET_ACCESS_KEY"],
+          :config_file => ((ENV["CONFIG_FILE"] && ENV["CONFIG_FILE"].empty?) ? "config/config.yml" : ENV["CONFIG_FILE"]),
+          :username => "root",
+          :ec2_dir => ENV["EC2_HOME"],
+          :keypair => ENV["KEYPAIR_NAME"],
+          :ami => 'ami-4a46a323',
+          :shared_bucket => "",
+          :services => "nginx sinatra",
+          :expand_when => "web_usage < 1.5\n memory_usage > 0.85",
+          :contract_when => "cpu_usage < 0.20\n memory_usage < 0.10",
+          :os => "ubuntu"
+        }
+      end
+      # Services monitored by Heartbeat
+      # Always at least monitors haproxy
+      def managed_services
+        "haproxy #{services}"
+      end
+      def launching_user_data
+        {:polling_time => polling_time}.to_yaml
+      end
+      # Keypair path
+      # Idiom:
+      #  /Users/username/.ec2/id_rsa-name
+      def keypair_path
+        "#{ec2_dir}/id_rsa-#{keypair}"
+      end
+      # Are we in development or test mode
+      def development?
+        environment == 'development'
+      end
+      # Are we in production mode?
+      def production?
+        environment == "production"
+      end
+      # Standard configuration files
+      %w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
+        define_method "#{file}_config_file" do
+          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
+        end
+      end
+      
+      # Call the options from the Application
+      def method_missing(m,*args)
+        options.methods.include?("#{m}") ? options.send(m,args) : super
+      end                         
+    end
+        
+  end
+    
+end

data/lib/pool_party/init.rb

@@ -0,0 +1,4 @@
+=begin rdoc
+  Load the files in order
+=end
+%w(optioner application monitors scheduler remoting os remote_instance master tasks).each {|f| require File.join(File.dirname(__FILE__), f)}

data/lib/pool_party/master.rb

@@ -0,0 +1,189 @@
+=begin rdoc
+  The basic master for PoolParty
+=end
+require "aska"
+module PoolParty
+  class Master < Remoting
+    include Aska
+    
+    def initialize
+      super
+      
+      self.class.send :rules, :contract_when, Application.options.contract_when
+      self.class.send :rules, :expand_when, Application.options.expand_when
+    end
+    # Start the cloud
+    def start_cloud!
+      start!
+    end
+    # Start the cloud, which launches the minimum_instances
+    def start!
+      message "Launching minimum_instances"
+      launch_minimum_instances
+      message "Waiting for master to boot up" 
+      reset!
+      while !number_of_pending_instances.zero?
+        wait "2.seconds"
+        reset!
+      end
+      message "Give some time for the instance ssh to start up"
+      wait "10.seconds"
+      message "Configuring master"
+      get_node(0).configure
+    end
+    # Launch the minimum number of instances. 
+    def launch_minimum_instances
+      request_launch_new_instances(Application.minimum_instances - number_of_pending_and_running_instances)
+      nodes
+    end
+    # Start monitoring the cloud with the threaded loop
+    def start_monitor!
+      begin
+        trap("INT") do
+          on_exit
+          exit
+        end
+        run_thread_loop(:daemonize => true) do
+          add_task {launch_minimum_instances} # If the base instances go down...
+          add_task {reconfigure_cloud_when_necessary}
+          add_task {add_instance_if_load_is_high}
+          add_task {terminate_instance_if_load_is_low}
+        end
+      rescue Exception => e
+        puts "There was an error: #{e.nice_message}"
+      end
+    end
+    # Tough method:
+    # We need to make sure that all the instances have the required software installed
+    # This is a basic check against the local store of the instances that have the 
+    # stack installed.
+    def reconfigure_cloud_when_necessary
+      reconfigure_running_instances if number_of_unconfigured_nodes > 0
+    end
+    def number_of_unconfigured_nodes
+      nodes.reject {|a| a.stack_installed? }.size
+    end
+    # Add an instance if the load is high
+    def add_instance_if_load_is_high      
+      request_launch_new_instance if expand?
+    end
+    # Teardown an instance if the load is pretty low
+    def terminate_instance_if_load_is_low
+      if contract?
+        node = nodes.reject {|a| a.master? }[-1]
+        request_termination_of_instance(node.instance_id) if node
+      end
+    end
+    # FOR MONITORING
+    def contract?
+      valid_rules?(:contract_when)
+    end
+    def expand?
+      valid_rules?(:expand_when)
+    end
+    # Get the average web requests per cloud
+    def web_requests
+      nodes.collect {|a| a.web } / nodes.size
+    end
+    # Get the average cpu usage per cloud
+    def cpu_usage
+      nodes.collect {|a| a.cpu } / nodes.size
+    end
+    # Get the average memory usage over the cloud
+    def memory_usage
+      nodes.collect {|a| a.memory } / nodes.size
+    end
+    # Restart the running instances services with monit on all the nodes
+    def restart_running_instances_services
+      nodes.each do |node|
+        node.restart_with_monit
+      end
+    end
+    # Reconfigure the running instances
+    def reconfigure_running_instances      
+      nodes.each do |node|
+        node.configure if node.status =~ /running/
+      end
+    end
+    # Build the basic haproxy config file from the config file in the config directory and return a tempfile
+    def build_haproxy_file
+      servers=<<-EOS        
+#{nodes.collect {|node| node.haproxy_entry}.join("\n")}
+      EOS
+      write_to_temp_file(open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port})
+    end
+    # Build the hosts file and return a tempfile
+    def build_hosts_file
+      write_to_temp_file(nodes.collect {|a| a.hosts_entry }.join("\n"))
+    end
+    # Build host file for a specific node
+    def build_hosts_file_for(n)
+      servers=<<-EOS        
+#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}
+      EOS
+      write_to_temp_file(servers)
+    end
+    # Build the basic auth file for the heartbeat
+    def build_heartbeat_authkeys_file
+      write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read)
+    end
+    # Return a list of the nodes and cache them
+    def nodes
+      @nodes ||= list_of_nonterminated_instances.collect_with_index do |inst, i|
+        RemoteInstance.new(inst.merge({:number => i}))
+      end
+    end
+    # Get the node at the specific index from the cached nodes
+    def get_node(i=0)
+      nodes.select {|a| a.number == i}.first
+    end
+    # Get the next node in sequence, so we can configure heartbeat to monitor the next node
+    def get_next_node(node)
+      i = node.number + 1
+      i = 0 if i >= (nodes.size - 1)
+      get_node(i)
+    end
+    # On exit command
+    def on_exit      
+    end
+    # List the clouds
+    def list
+      if number_of_pending_and_running_instances > 0
+        out = "-- CLOUD (#{number_of_pending_and_running_instances})--\n"
+        out << nodes.collect {|node| node.description }.join("\n")
+      else
+        out = "Cloud is not running"
+      end
+      out
+    end
+    # Reset and clear the caches
+    def reset!
+      @cached_descriptions = nil
+      @nodes = nil
+    end
+    
+    class << self
+      def requires_heartbeat?
+        new.nodes.size > 1
+      end
+      def get_next_node(node)
+        new.get_next_node(node)
+      end
+      # Build a heartbeat_config_file from the config file in the config directory and return a tempfile
+      def build_heartbeat_config_file_for(node)
+        return nil unless node
+        servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}"
+        write_to_temp_file(open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers})
+      end
+      # Build a heartbeat resources file from the config directory and return a tempfile
+      def build_heartbeat_resources_file_for(node)
+        return nil unless node
+        write_to_temp_file("#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}")
+      end
+      def build_hosts_file_for(node)
+        new.build_hosts_file_for(node)
+      end
+    end
+    
+  end
+end

data/lib/pool_party/monitors/cpu.rb

@@ -0,0 +1,18 @@
+=begin rdoc
+  Basic monitor on the cpu stats
+=end
+module PoolParty
+  module Monitors
+    module Cpu
+      def self.monitor!
+        IO.popen("uptime") do |up|
+          ret = monitor_from_string(up)
+        end
+        ret
+      end
+      def self.monitor_from_string(str="")
+        str.split(/\s+/)[-3].to_f
+      end
+    end
+  end
+end

data/lib/pool_party/monitors/memory.rb

@@ -0,0 +1,21 @@
+=begin rdoc
+  Basic memory monitor on the instance
+=end
+module PoolParty
+  module Monitors
+    module Memory
+      def self.monitor!
+        IO.popen("free -m | grep -i mem") { |io|
+          ret = monitor_from_string(io)
+        }
+        ret
+      end
+      def self.monitor_from_string(str="")
+        total_memory = str.split[1].to_f
+        used_memory = str.split[2].to_f
+        
+        used_memory / total_memory
+      end
+    end
+  end
+end

data/lib/pool_party/monitors/web.rb

@@ -0,0 +1,18 @@
+=begin rdoc
+  Monitor the web stats for the request rate the server can handle at a time
+=end
+module PoolParty
+  module Monitors
+    module Web
+      def self.monitor!(port)
+        IO.popen("httperf --server localhost --port #{port} --num-conn 3 --timeout 5 | grep 'Request rate'") do |io|
+          @req = monitor_from_string(io.gets)
+        end
+        @req
+      end
+      def self.monitor_from_string(str="")
+        str[/[.]* ([\d]*\.[\d]*) [.]*/, 0].chomp.to_f
+      end
+    end
+  end
+end