poolparty 0.0.4 → 0.2.6

data/config/heartbeat_authkeys.conf

@@ -1,2 +0,0 @@
-auth 1
-1 md5 PasswordForPoolPartyClusterApplicationButItIsOnlyADefault

data/config/monit.conf

@@ -1,8 +0,0 @@
-# Configuration for Monit, a monitoring tool
-set daemon 20 # Perform a check every 20 seconds
-set logfile /var/log/monit.log
-
-set httpd port 2812 and
-    use address 0.0.0.0
-
-include /etc/monit.d/*

data/config/monit/haproxy.monit.conf

@@ -1,7 +0,0 @@
-check process haproxy with pidfile /var/run/haproxy.pid
-  start program = "/usr/sbin/haproxy -f /etc/haproxy.cfg -p /var/run/haproxy.pid"
-  stop program  = "/usr/bin/killall -9 haproxy"
-  if totalmem is greater than 100.0 MB for 4 cycles then restart
-  if cpu is greater than 50% for 2 cycles then alert
-  if cpu is greater than 80% for 3 cycles then restart
-  if loadavg(5min) greater than 10 for 8 cycles then restart

data/config/nginx.conf

@@ -1,24 +0,0 @@
-user  usr usr;
-worker_processes  2;
-
-http {
-    sendfile       on;
-    tcp_nopush     on;
-
-    keepalive_timeout  65;
-    tcp_nodelay        on;
-
-    upstream fast_mongrels  { server 127.0.0.1:8100; }
-
-    server {
-        listen       80;
-        server_name  srv;
-        root        /home/usr;
-     
-     location / {
-         proxy_pass http://fast_mongrels;
-         break;
-     }
-      
-  }
-}

data/lib/core/array.rb

@@ -1,10 +0,0 @@
-=begin rdoc
-  Array extensions
-=end
-require "enumerator"
-class Array
-  # Collection with the index
-  def collect_with_index &block
-    self.enum_for(:each_with_index).collect &block
-  end
-end

data/lib/core/kernel.rb

@@ -1,9 +0,0 @@
-=begin rdoc
-  Kernel overloads
-=end
-module Kernel
-  # Nice wait instead of sleep
-  def wait(time=10)
-    sleep time.is_a?(String) ? eval(time) : time
-  end
-end

data/lib/core/object.rb

@@ -1,14 +0,0 @@
-=begin rdoc
-  Basic, add an alias_method to the object class
-  Add returning to the object
-=end
-class Object
-  def alias_method(new_id, original_id)
-    original = self.method(original_id).to_proc
-    define_method(new_id){|*args| original.call(*args)}
-  end
-  def returning(receiver)
-    yield receiver
-    receiver
-  end
-end

data/lib/modules/callback.rb

@@ -1,55 +0,0 @@
-module PoolParty
-  module Callbacks
-    module ClassMethods
-      def callback(type,m,e,*args, &block)
-        
-        # Save the old method
-        if method_defined?("#{type}_#{m}".to_sym)
-          puts "method already defined"
-        end
-                
-        case type
-        when :before          
-          str=<<-EOD
-            def #{type}_#{m}
-              #{e}              
-              yield if block_given?
-              super
-            end
-          EOD
-        when :after
-          str=<<-EOD
-            def #{type}_#{m}
-              super
-              yield if block_given?
-              #{e}
-            end
-          EOD
-        end
-                
-        mMod = Module.new {eval str}
-        
-        module_eval %{alias_method :#{type}_#{m}, :#{m}}
-        
-        self.send :define_method, "#{m}".to_sym, Proc.new {
-          extend(mMod)
-          method("#{type}_#{m}".to_sym).call
-        }
-      end
-      def before(m,e,*args, &block)
-        callback(:before,m,e,*args, &block)
-      end
-      def after(m,e,*args, &block)
-        callback(:after,m,e,*args,&block)
-      end
-    end
-    
-    module InstanceMethods            
-    end
-    
-    def self.included(receiver)
-      receiver.extend         ClassMethods
-      receiver.send :include, InstanceMethods
-    end
-  end
-end

data/lib/modules/ec2_wrapper.rb

@@ -1,74 +0,0 @@
-module PoolParty
-  extend self
-  
-  module Ec2Wrapper
-    
-    module ClassMethods      
-    end
-    
-    module InstanceMethods
-      # Run a new instance, with the user_data and the ami described in the config
-      def launch_new_instance!
-        instance = ec2.run_instances(
-          :image_id => Application.ami, 
-          :user_data => "#{Application.launching_user_data}",
-          :minCount => 1,
-          :maxCount => 1,
-          :key_name => Application.keypair,
-          :size => "#{Application.size}")
-          
-        item = instance.RunInstancesResponse.instancesSet.item
-        EC2ResponseObject.get_hash_from_response(item)
-      end
-      # Shutdown the instance by instance_id
-      def terminate_instance!(instance_id)
-        ec2.terminate_instances(:instance_id => instance_id)
-      end
-      # Instance description
-      def describe_instance(id)
-        instance = ec2.describe_instances(:instance_id => id)
-        item = instance.DescribeInstancesResponse.reservationSet.item.instancesSet.item
-        EC2ResponseObject.get_hash_from_response(item)
-      end
-      # Get instance by id
-      def get_instance_by_id(id)
-        get_instances_description.select {|a| a.instance_id == id}[0] rescue nil
-      end
-      # Get the s3 description for the response in a hash format
-      def get_instances_description
-        @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
-      end
-            
-      # EC2 connections
-      def ec2
-        @ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
-      end      
-    end
-    
-    def self.included(receiver)
-      receiver.extend         ClassMethods
-      receiver.send :include, InstanceMethods
-    end
-  end
-  # Provides a simple class to wrap around the amazon responses
-  class EC2ResponseObject
-    def self.get_descriptions(resp)
-      rs = resp.DescribeInstancesResponse.reservationSet.item
-      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
-      out = begin
-        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
-      rescue Exception => e
-        begin
-          # Really weird bug with amazon's ec2 gem
-          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
-        rescue Exception => e
-          []
-        end                
-      end
-      out
-    end
-    def self.get_hash_from_response(resp)
-      {:instance_id => resp.instanceId, :ip => resp.dnsName, :status => resp.instanceState.name, :launching_time => resp.launchTime} rescue nil
-    end
-  end
-end

data/lib/pool_party.rb

@@ -1,69 +0,0 @@
-=begin rdoc
-  The main file, contains the client and the server application methods
-=end
-$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
-
-# rubygems
-require 'rubygems'
-require "aws/s3"
-require "sqs"
-require "EC2"
-require 'thread'
-require "pp"
-require "tempfile"
-require "aska"
-begin
-  require 'fastthread'
-  require 'thin'
-rescue LoadError
-end
-
-## Load PoolParty
-pwd = File.dirname(__FILE__)
-
-# Load the required files
-# If there is an init file, load that, otherwise
-# require all the files in each directory
-%w(core modules s3 pool_party).each do |dir|  
-  Dir["#{pwd}/#{dir}"].each do |dir|
-    begin
-      require File.join(dir, "init")
-    rescue LoadError => e
-      Dir["#{pwd}/#{File.basename(dir)}/**"].each {|file| require File.join(dir, File.basename(file))}
-    end
-  end
-end
-
-module PoolParty  
-  module Version
-    MAJOR = '0'
-    MINOR = '0'
-    REVISION = '4'
-    def self.combined
-      [MAJOR, MINOR, REVISION].join('.')
-    end
-  end
-  # PoolParty options
-  def options(opts={})
-    Application.options(opts)
-  end
-  # Are we working in verbose-mode
-  def verbose?
-    Application.verbose == true
-  end
-  # Send a message if we are in verbose-mode
-  def message(msg="")
-    pp "-- #{msg}" if verbose?
-  end
-  # Root directory of the application
-  def root_dir
-    File.dirname(__FILE__)
-  end
-  # Write string to a tempfile
-  def write_to_temp_file(str="")
-    tempfile = Tempfile.new("rand#{rand(1000)}-#{rand(1000)}")
-    tempfile.print(str)
-    tempfile.flush
-    tempfile
-  end
-end

data/lib/pool_party/application.rb

@@ -1,133 +0,0 @@
-=begin rdoc
-  Application
-  This handles user interaction
-=end
-$:.unshift File.dirname(__FILE__)
-
-module PoolParty
-  extend self
-  
-  class Application
-    class << self
-      
-      # The application options
-      def options(opts={})
-        @options ||= make_options(opts)
-      end      
-      # Make the options with the config_file overrides included
-      # Default config file assumed to be at config/config.yml
-      def make_options(opts={})
-        load_options!
-        default_options.merge!(opts)
-        # If the config_file options are specified and not empty
-        unless default_options[:config_file].nil? || default_options[:config_file].empty?
-          require "yaml"
-          # Try loading the file if it exists
-          filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
-          default_options.merge!( YAML.load(filedata) ) if filedata
-        end
-
-        OpenStruct.new(default_options)
-      end
-
-      # Load options via commandline
-      def load_options!
-        require 'optparse'
-        OptionParser.new do |op|
-          op.on('-A key', '--access-key key', "Ec2 access key (ENV['ACCESS_KEY'])") { |key| default_options[:access_key] = key }
-          op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['SECRET_ACCESS_KEY'])") { |key| default_options[:secret_access_key] = key }
-          op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-4a46a323')") {|id| default_options[:ami] = id }
-          op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
-          op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }          
-          op.on('-D ec2 directory', '--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
-          op.on('-S services', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
-          op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
-          op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
-          op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }          
-          op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
-          op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }          
-          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
-          op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
-          op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
-          op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
-          op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
-          op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = v}
-          op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i}
-          op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x}
-
-          op.on_tail("-h", "--help", "Show this message") do |o|
-            puts "op: #{o}"
-            exit
-          end          
-        end.parse!(ARGV.dup)
-      end
-      
-      # Basic default options
-      # All can be overridden by the command line
-      # or in a config.yml file
-      def default_options
-        @default_options ||= {
-          :host_port => 80,
-          :client_port => 8001,
-          :environment => 'development',
-          :verbose => true,
-          :logging => true,
-          :size => "small",
-          :polling_time => "30.seconds",
-          :user_data => "",
-          :heavy_load => 0.80,
-          :light_load => 0.15,
-          :minimum_instances => 1,
-          :maximum_instances => 3,
-          :access_key => ENV["ACCESS_KEY"],
-          :secret_access_key => ENV["SECRET_ACCESS_KEY"],
-          :config_file => ((ENV["CONFIG_FILE"] && ENV["CONFIG_FILE"].empty?) ? "config/config.yml" : ENV["CONFIG_FILE"]),
-          :username => "root",
-          :ec2_dir => ENV["EC2_HOME"],
-          :keypair => ENV["KEYPAIR_NAME"],
-          :ami => 'ami-4a46a323',
-          :shared_bucket => "",
-          :services => "nginx sinatra",
-          :expand_when => "web_usage < 1.5\n memory_usage > 0.85",
-          :contract_when => "cpu_usage < 0.20\n memory_usage < 0.10",
-          :os => "ubuntu"
-        }
-      end
-      # Services monitored by Heartbeat
-      # Always at least monitors haproxy
-      def managed_services
-        "haproxy #{services}"
-      end
-      def launching_user_data
-        {:polling_time => polling_time}.to_yaml
-      end
-      # Keypair path
-      # Idiom:
-      #  /Users/username/.ec2/id_rsa-name
-      def keypair_path
-        "#{ec2_dir}/id_rsa-#{keypair}"
-      end
-      # Are we in development or test mode
-      def development?
-        environment == 'development'
-      end
-      # Are we in production mode?
-      def production?
-        environment == "production"
-      end
-      # Standard configuration files
-      %w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
-        define_method "#{file}_config_file" do
-          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
-        end
-      end
-      
-      # Call the options from the Application
-      def method_missing(m,*args)
-        options.methods.include?("#{m}") ? options.send(m,args) : super
-      end                         
-    end
-        
-  end
-    
-end

data/lib/pool_party/init.rb

@@ -1,4 +0,0 @@
-=begin rdoc
-  Load the files in order
-=end
-%w(optioner application monitors scheduler remoting os remote_instance master tasks).each {|f| require File.join(File.dirname(__FILE__), f)}

data/lib/pool_party/master.rb

@@ -1,189 +0,0 @@
-=begin rdoc
-  The basic master for PoolParty
-=end
-require "aska"
-module PoolParty
-  class Master < Remoting
-    include Aska
-    
-    def initialize
-      super
-      
-      self.class.send :rules, :contract_when, Application.options.contract_when
-      self.class.send :rules, :expand_when, Application.options.expand_when
-    end
-    # Start the cloud
-    def start_cloud!
-      start!
-    end
-    # Start the cloud, which launches the minimum_instances
-    def start!
-      message "Launching minimum_instances"
-      launch_minimum_instances
-      message "Waiting for master to boot up" 
-      reset!
-      while !number_of_pending_instances.zero?
-        wait "2.seconds"
-        reset!
-      end
-      message "Give some time for the instance ssh to start up"
-      wait "10.seconds"
-      message "Configuring master"
-      get_node(0).configure
-    end
-    # Launch the minimum number of instances. 
-    def launch_minimum_instances
-      request_launch_new_instances(Application.minimum_instances - number_of_pending_and_running_instances)
-      nodes
-    end
-    # Start monitoring the cloud with the threaded loop
-    def start_monitor!
-      begin
-        trap("INT") do
-          on_exit
-          exit
-        end
-        run_thread_loop(:daemonize => true) do
-          add_task {launch_minimum_instances} # If the base instances go down...
-          add_task {reconfigure_cloud_when_necessary}
-          add_task {add_instance_if_load_is_high}
-          add_task {terminate_instance_if_load_is_low}
-        end
-      rescue Exception => e
-        puts "There was an error: #{e.nice_message}"
-      end
-    end
-    # Tough method:
-    # We need to make sure that all the instances have the required software installed
-    # This is a basic check against the local store of the instances that have the 
-    # stack installed.
-    def reconfigure_cloud_when_necessary
-      reconfigure_running_instances if number_of_unconfigured_nodes > 0
-    end
-    def number_of_unconfigured_nodes
-      nodes.reject {|a| a.stack_installed? }.size
-    end
-    # Add an instance if the load is high
-    def add_instance_if_load_is_high      
-      request_launch_new_instance if expand?
-    end
-    # Teardown an instance if the load is pretty low
-    def terminate_instance_if_load_is_low
-      if contract?
-        node = nodes.reject {|a| a.master? }[-1]
-        request_termination_of_instance(node.instance_id) if node
-      end
-    end
-    # FOR MONITORING
-    def contract?
-      valid_rules?(:contract_when)
-    end
-    def expand?
-      valid_rules?(:expand_when)
-    end
-    # Get the average web requests per cloud
-    def web_requests
-      nodes.collect {|a| a.web } / nodes.size
-    end
-    # Get the average cpu usage per cloud
-    def cpu_usage
-      nodes.collect {|a| a.cpu } / nodes.size
-    end
-    # Get the average memory usage over the cloud
-    def memory_usage
-      nodes.collect {|a| a.memory } / nodes.size
-    end
-    # Restart the running instances services with monit on all the nodes
-    def restart_running_instances_services
-      nodes.each do |node|
-        node.restart_with_monit
-      end
-    end
-    # Reconfigure the running instances
-    def reconfigure_running_instances      
-      nodes.each do |node|
-        node.configure if node.status =~ /running/
-      end
-    end
-    # Build the basic haproxy config file from the config file in the config directory and return a tempfile
-    def build_haproxy_file
-      servers=<<-EOS        
-#{nodes.collect {|node| node.haproxy_entry}.join("\n")}
-      EOS
-      write_to_temp_file(open(Application.haproxy_config_file).read.strip ^ {:servers => servers, :host_port => Application.host_port})
-    end
-    # Build the hosts file and return a tempfile
-    def build_hosts_file
-      write_to_temp_file(nodes.collect {|a| a.hosts_entry }.join("\n"))
-    end
-    # Build host file for a specific node
-    def build_hosts_file_for(n)
-      servers=<<-EOS        
-#{nodes.collect {|node| node.ip == n.ip ? node.local_hosts_entry : node.hosts_entry}.join("\n")}
-      EOS
-      write_to_temp_file(servers)
-    end
-    # Build the basic auth file for the heartbeat
-    def build_heartbeat_authkeys_file
-      write_to_temp_file(open(Application.heartbeat_authkeys_config_file).read)
-    end
-    # Return a list of the nodes and cache them
-    def nodes
-      @nodes ||= list_of_nonterminated_instances.collect_with_index do |inst, i|
-        RemoteInstance.new(inst.merge({:number => i}))
-      end
-    end
-    # Get the node at the specific index from the cached nodes
-    def get_node(i=0)
-      nodes.select {|a| a.number == i}.first
-    end
-    # Get the next node in sequence, so we can configure heartbeat to monitor the next node
-    def get_next_node(node)
-      i = node.number + 1
-      i = 0 if i >= (nodes.size - 1)
-      get_node(i)
-    end
-    # On exit command
-    def on_exit      
-    end
-    # List the clouds
-    def list
-      if number_of_pending_and_running_instances > 0
-        out = "-- CLOUD (#{number_of_pending_and_running_instances})--\n"
-        out << nodes.collect {|node| node.description }.join("\n")
-      else
-        out = "Cloud is not running"
-      end
-      out
-    end
-    # Reset and clear the caches
-    def reset!
-      @cached_descriptions = nil
-      @nodes = nil
-    end
-    
-    class << self
-      def requires_heartbeat?
-        new.nodes.size > 1
-      end
-      def get_next_node(node)
-        new.get_next_node(node)
-      end
-      # Build a heartbeat_config_file from the config file in the config directory and return a tempfile
-      def build_heartbeat_config_file_for(node)
-        return nil unless node
-        servers = "#{node.node_entry}\n#{get_next_node(node).node_entry}"
-        write_to_temp_file(open(Application.heartbeat_config_file).read.strip ^ {:nodes => servers})
-      end
-      # Build a heartbeat resources file from the config directory and return a tempfile
-      def build_heartbeat_resources_file_for(node)
-        return nil unless node
-        write_to_temp_file("#{node.haproxy_resources_entry}\n#{get_next_node(node).haproxy_resources_entry}")
-      end
-      def build_hosts_file_for(node)
-        new.build_hosts_file_for(node)
-      end
-    end
-    
-  end
-end