dreadpiratepj-poolparty 0.0.8

data/lib/modules/callback.rb

@@ -0,0 +1,133 @@
+=begin rdoc
+  Basic callbacks
+=end
+module PoolParty
+  module Callbacks
+    module ClassMethods
+      def define_callback_module(mod)
+        callbacks << mod
+      end
+      def define_callback_class(cla)
+        classes << cla
+      end
+      def callback(type, m, *args, &block)
+        arr = []                
+        args.each do |arg|
+          arr << case arg.class.to_s
+          when "Hash"
+            arg.collect do |meth, klass|
+              case klass.class.to_s
+              when "String"
+                define_callback_class(klass)
+                "self.#{klass.to_s.downcase}.#{meth}(self)"
+              else
+                "#{klass}.send :#{meth}, self"
+              end              
+            end
+          when "Symbol"
+            "self.send :#{arg}, self"
+          end
+        end
+
+        string = ""
+        if block_given?
+          num = store_proc(block.to_proc)
+          arr << <<-EOM
+            self.class.get_proc(#{num}).bind(self).call
+          EOM
+        end
+
+        string = create_eval_for_mod_with_string_and_type!(m, type) do
+          arr.join("\n")
+        end        
+
+        mMode = Module.new {eval string}
+
+        define_callback_module(mMode)
+      end
+      def before(m, *args, &block)
+        callback(:before, m, *args, &block)
+      end
+      def after(m, *args, &block)
+        callback(:after, m, *args, &block)
+      end
+
+      def create_eval_for_mod_with_string_and_type!(meth, type=nil, &block)
+        str = ""
+        case type
+        when :before          
+          str << <<-EOD
+            def #{meth}(*args)
+              #{yield}
+              super
+            end
+          EOD
+        when :after
+          str << <<-EOD
+            def #{meth}(*args)
+              super
+              #{yield}
+            end
+          EOD
+        else
+          str << <<-EOD
+            def #{meth}(*args)
+              #{yield}
+            end
+          EOD
+        end
+        str
+      end
+
+      def callbacks
+        @callbacks ||= []
+      end
+      def classes
+        @classes ||= []
+      end
+    end
+
+    module InstanceMethods
+      def initialize(*args)
+        extend_callbacks
+        extend_callback_methods
+      end
+      
+      def extend_callback_methods
+        unless self.class.classes.empty? 
+          self.class.classes.each do |klass|
+            m = %{def #{klass.to_s.downcase};@#{klass.to_s.downcase} ||= #{klass}.new;end}
+            self.class.class_eval m unless self.class.method_defined?(m)
+          end
+        end
+      end
+      
+      def extend_callbacks
+        unless self.class.callbacks.empty?
+          self.class.callbacks.each do |mod|
+            self.extend(mod)
+          end
+        end
+      end
+    end
+
+    module ProcStoreMethods
+      def store_proc(proc)
+        proc_storage << proc
+        proc_storage.index(proc)
+      end
+      def get_proc(num)
+        proc_storage[num]
+      end
+      def proc_storage
+        @proc_store ||= []
+      end
+    end
+
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.extend         ProcStoreMethods
+      receiver.send :include, InstanceMethods
+    end    
+  end
+end

data/lib/modules/ec2_wrapper.rb

@@ -0,0 +1,82 @@
+module PoolParty
+  extend self
+  
+  module Ec2Wrapper
+    
+    module ClassMethods      
+    end
+    
+    module InstanceMethods
+      # Run a new instance, with the user_data and the ami described in the config
+      def launch_new_instance!
+        instance = ec2.run_instances(
+          :image_id => Application.ami, 
+          :user_data => "#{Application.launching_user_data}",
+          :minCount => 1,
+          :maxCount => 1,
+          :key_name => Application.keypair,
+          :size => "#{Application.size}")
+          
+        item = instance.RunInstancesResponse.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Shutdown the instance by instance_id
+      def terminate_instance!(instance_id)
+        ec2.terminate_instances(:instance_id => instance_id)
+      end
+      def associate_address_with(ip, instance_id)
+        ec2.associate_address(:instance_id => instance_id, :public_ip => ip)
+      end
+      # Instance description
+      def describe_instance(id)
+        instance = ec2.describe_instances(:instance_id => id)
+        item = instance.DescribeInstancesResponse.reservationSet.item.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Get instance by id
+      def get_instance_by_id(id)
+        get_instances_description.select {|a| a.instance_id == id}[0] rescue nil
+      end
+      # Get the s3 description for the response in a hash format
+      def get_instances_description
+        @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
+      end
+            
+      # EC2 connections
+      def ec2
+        @ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
+      end      
+    end
+    
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+  # Provides a simple class to wrap around the amazon responses
+  class EC2ResponseObject
+    def self.get_descriptions(resp)
+      rs = resp.DescribeInstancesResponse.reservationSet.item
+      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      out = begin
+        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
+      rescue Exception => e
+        begin
+          # Really weird bug with amazon's ec2 gem
+          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
+        rescue Exception => e
+          []
+        end                
+      end
+      out
+    end
+    def self.get_hash_from_response(resp)
+      {
+        :instance_id => resp.instanceId, 
+        :ip => resp.dnsName, 
+        :status => resp.instanceState.name, 
+        :launching_time => resp.launchTime
+      } rescue nil
+    end
+  end
+end

data/lib/modules/safe_instance.rb

@@ -0,0 +1,31 @@
+=begin rdoc
+  Make a command thread-safe
+=end
+require "monitor"
+module PoolParty
+  extend self
+  
+  module ThreadSafeInstance
+    
+    module ClassMethods
+      def make_safe(meth)
+       original_method = "_unsafe_#{meth}_"
+       alias_method original_method, meth
+       define_method(meth) {|*args| self.class.synchronize { self.send(original_method) } }
+       self
+      end  
+    end
+    
+    module InstanceMethods
+      def make_safe meth
+        self.class.make_safe meth
+      end
+    end
+    
+    def self.included(receiver)
+      receiver.extend         MonitorMixin
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end

data/lib/modules/vlad_override.rb

@@ -0,0 +1,82 @@
+require "vlad"
+class Rake::RemoteTask < Rake::Task
+  def run command
+    cmd = [ssh_cmd, ssh_flags, target_host, command].compact
+    result = []
+    
+    warn cmd.join(' ') if $TRACE
+
+    pid, inn, out, err = popen4(*cmd.join(" "))
+
+    inn.sync   = true
+    streams    = [out, err]
+    out_stream = {
+      out => $stdout,
+      err => $stderr,
+    }
+
+    # Handle process termination ourselves
+    status = nil
+    Thread.start do
+      status = Process.waitpid2(pid).last
+    end
+
+    until streams.empty? do
+      # don't busy loop
+      selected, = select streams, nil, nil, 0.1
+
+      next if selected.nil? or selected.empty?
+
+      selected.each do |stream|
+        if stream.eof? then
+          streams.delete stream if status # we've quit, so no more writing
+          next
+        end
+
+        data = stream.readpartial(1024)
+        out_stream[stream].write data
+
+        if stream == err and data =~ /^Password:/ then
+          inn.puts sudo_password
+          data << "\n"
+          $stderr.write "\n"
+        end
+
+        result << data
+      end
+    end
+
+    PoolParty.message "execution failed with status #{status.exitstatus}: #{cmd.join ' '}" unless status.success?
+
+    result.join
+  end
+  
+  def rsync local, remote
+    cmd = [rsync_cmd, rsync_flags, local, "#{@target_host}:#{remote}"].flatten.compact
+  
+    success = system(*cmd.join(" "))
+
+    unless success then
+      raise Vlad::CommandFailedError, "execution failed: #{cmd.join ' '}"
+    end
+  end
+  def set name, val = nil, &b
+    rt.set name, val, &b
+  end
+  def rt
+    @rt ||= Rake::RemoteTask
+  end
+  
+  def target_hosts
+    if hosts = ENV["HOSTS"] then
+      hosts.strip.gsub(/\s+/, '').split(",")
+    elsif options[:single]
+      @roles = {}; @roles[:app] = {}      
+      @roles[:app][options[:single]] = options[:single]
+      roles = Rake::RemoteTask.hosts_for(@roles)
+    else
+      roles = options[:roles]
+      roles ? Rake::RemoteTask.hosts_for(roles) : Rake::RemoteTask.all_hosts
+    end
+  end
+end

data/lib/poolparty.rb

@@ -0,0 +1,105 @@
+=begin rdoc
+  The main file, contains the client and the server application methods
+=end
+$:.unshift File.dirname(__FILE__)     # For use/testing when no gem is installed
+
+# rubygems
+require 'rubygems'
+require "aws/s3"
+require "sqs"
+require "EC2"
+require 'thread'
+require "pp"
+require "tempfile"
+require "aska"
+require "vlad"
+begin
+  require 'fastthread'
+  require 'thin'
+  require 'system_timer'
+  Timer = SystemTimer  
+rescue LoadError
+  require 'timeout'
+  Timer = Timeout
+end
+
+## Load PoolParty
+pwd = File.dirname(__FILE__)
+
+# Load the required files
+# If there is an init file, load that, otherwise
+# require all the files in each directory
+%w(core modules s3 poolparty).each do |dir|  
+  Dir["#{pwd}/#{dir}"].each do |dir|
+    begin
+      require File.join(dir, "init")
+    rescue LoadError => e
+      Dir["#{pwd}/#{File.basename(dir)}/**"].each {|file| require File.join(dir, File.basename(file))}
+    end
+  end
+end
+
+module PoolParty
+  module Version #:nodoc:
+    MAJOR = 0
+    MINOR = 0
+    TINY  = 8
+
+    STRING = [MAJOR, MINOR, TINY].join('.')
+  end
+  # PoolParty options
+  def options(opts={})
+    Application.options(opts)
+  end
+  # Are we working in verbose-mode
+  def verbose?
+    options.verbose == true
+  end
+  # Send a message if we are in verbose-mode
+  def message(msg="")
+    puts "-- #{msg}" if verbose?
+  end
+  # Root directory of the application
+  def root_dir
+    File.expand_path(File.dirname(__FILE__) + "/..")
+  end
+  # User directory
+  def user_dir
+    Dir.pwd
+  end
+  # Write string to a tempfile
+  def write_to_temp_file(str="")
+    tempfile = Tempfile.new("rand#{rand(1000)}-#{rand(1000)}")
+    tempfile.print(str)
+    tempfile.flush
+    tempfile
+  end
+  def register_monitor(*names)
+    names.each do |name|
+      PoolParty::Monitors.extend name
+      
+      PoolParty::Master.send :include, name::Master
+      PoolParty::RemoteInstance.send :include, name::Remote
+    end
+  end
+  def load_plugins
+    Dir["#{plugin_dir}/**/init.rb"].each {|a| require a}
+  end
+  def reset!
+    @@installed_plugins = nil
+    Application.options = nil
+  end
+  def plugin_dir
+    "#{user_dir}/vendor"
+  end
+  def read_config_file(filename)
+    return {} unless filename
+    YAML.load(open(filename).read)
+  end
+  def include_cloud_tasks
+    Tasks.new.define_tasks
+  end
+  
+  alias_method :tasks, :include_cloud_tasks
+  alias_method :include_tasks, :include_cloud_tasks
+end

data/lib/poolparty/application.rb

@@ -0,0 +1,170 @@
+=begin rdoc
+  Application
+  This handles user interaction
+=end
+module PoolParty
+  class Application
+    class << self
+      attr_accessor :verbose, :options
+      
+      # The application options
+      def options(opts={})
+        @options ||= make_options(opts)        
+      end      
+      # Make the options with the config_file overrides included
+      # Default config file assumed to be at config/config.yml
+      def make_options(opts={})
+        loading_options = opts.delete(:optsparse) || {}
+        loading_options.merge!( {:argv => opts.delete(:argv)} )
+        
+        load_options!(loading_options)
+        # default_options.merge!(opts)
+        # If the config_file options are specified and not empty
+        unless default_options[:config_file].nil? || default_options[:config_file].empty?
+          require "yaml"
+          # Try loading the file if it exists
+          filedata = open(default_options[:config_file]).read if File.file?(default_options[:config_file])
+          default_options.merge!( YAML.load(filedata) ) if filedata # We want the command-line to overwrite the config file
+        end
+        
+        OpenStruct.new(default_options.merge(opts))
+      end
+
+      # Load options via commandline
+      def load_options!(opts={})
+        require 'optparse'
+        OptionParser.new do |op|
+          op.banner = opts[:banner] if opts[:banner]
+          op.on('-A key', '--access-key key', "Ec2 access key (ENV['ACCESS_KEY'])") { |key| default_options[:access_key] = key }
+          op.on('-S key', '--secret-access-key key', "Ec2 secret access key (ENV['SECRET_ACCESS_KEY'])") { |key| default_options[:secret_access_key] = key }
+          op.on('-I ami', '--image-id id', "AMI instance (default: 'ami-4a46a323')") {|id| default_options[:ami] = id }
+          op.on('-k keypair', '--keypair name', "Keypair name (ENV['KEYPAIR_NAME'])") { |key| default_options[:keypair] = key }
+          op.on('-b bucket', '--bucket bucket', "Application bucket") { |bucket| default_options[:shared_bucket] = bucket }          
+          op.on('-D ec2 directory', '--ec2-dir dir', "Directory with ec2 data (default: '~/.ec2')") {|id| default_options[:ec2_dir] = id }
+          op.on('-r names', '--services names', "Monitored services (default: '')") {|id| default_options[:services] = id }
+          op.on('-c file', '--config-file file', "Config file (default: '')") {|file| default_options[:config_file] = file }
+          op.on('-l plugin_dir', '--plugin-dir dir', "Plugin directory (default: '')") {|file| default_options[:plugin_dir] = file }
+          op.on('-p port', '--host_port port', "Run on specific host_port (default: 7788)") { |host_port| default_options[:host_port] = host_port }
+          op.on('-m monitors', '--monitors names', "Monitor instances using (default: 'web,memory,cpu')") {|s| default_options[:monitor_load_on] = s }          
+          op.on('-o port', '--client_port port', "Run on specific client_port (default: 7788)") { |client_port| default_options[:client_port] = client_port }
+          op.on('-O os', '--os os', "Configure for os (default: ubuntu)") { |os| default_options[:os] = os }          
+          op.on('-e env', '--environment env', "Run on the specific environment (default: development)") { |env| default_options[:env] = env }
+          op.on('-a address', '--public-ip address', "Associate this public address with the master node") {|s| default_options[:public_ip] = s}
+          op.on('-s size', '--size size', "Run specific sized instance") {|s| default_options[:size] = s}
+          op.on('-n name', '--name name', "Application name") {|n| default_options[:app_name] = n}
+          op.on('-u username', '--username name', "Login with the user (default: root)") {|s| default_options[:user] = s}
+          op.on('-d user-data','--user-data data', "Extra data to send each of the instances (default: "")") { |data| default_options[:user_data] = data }
+          op.on('-t seconds', '--polling-time', "Time between polling in seconds (default 50)") {|t| default_options[:polling_time] = t }
+          op.on('-v', '--[no-]verbose', 'Run verbosely (default: false)') {|v| default_options[:verbose] = true}
+          op.on('-i number', '--minimum-instances', "The minimum number of instances to run at all times (default 1)") {|i| default_options[:minimum_instances] = i.to_i}
+          op.on('-x number', '--maximum-instances', "The maximum number of instances to run (default 3)") {|x| default_options[:maximum_instances] = x.to_i}
+          
+          op.on_tail("-V", "Show version") do
+            puts Application.version
+            exit
+          end
+          op.on_tail("-h", "-?", "Show this message") do
+            puts op
+            exit
+          end
+        end.parse!(opts[:argv] ? opts.delete(:argv) : ARGV.dup)
+      end
+      
+      # Basic default options
+      # All can be overridden by the command line
+      # or in a config.yml file
+      def default_options
+        @default_options ||= {
+          :app_name => "application_name",
+          :host_port => 80,
+          :client_port => 8001,
+          :environment => 'development',
+          :verbose => false,
+          :logging => true,
+          :size => "m1.small",
+          :polling_time => "30.seconds",
+          :user_data => "",
+          :heavy_load => 0.80,
+          :light_load => 0.15,
+          :minimum_instances => 2,
+          :maximum_instances => 4,
+          :public_ip => "",
+          :access_key => ENV["AWS_ACCESS_KEY_ID"],
+          :secret_access_key => ENV["AWS_SECRET_ACCESS_ID"],
+          :config_file => if ENV["CONFIG_FILE"] && !ENV["CONFIG_FILE"].empty?
+            ENV["CONFIG_FILE"]
+          elsif File.file?("config/config.yml")
+            "config/config.yml"
+          else
+            nil
+          end,
+          :username => "root",
+          :ec2_dir => ENV["EC2_HOME"],
+          :keypair => ENV["KEYPAIR_NAME"],
+          :ami => 'ami-4a46a323',
+          :shared_bucket => "",
+          :services => "",
+          :expand_when => "web_usage < 1.5\n memory > 0.85",
+          :contract_when => "cpu < 0.20\n memory < 0.10",
+          :os => "ubuntu",
+          :plugin_dir => "vendor",
+          :install_on_load => true
+        }
+      end
+      # Services monitored by Heartbeat
+      # Always at least monitors haproxy
+      def managed_services
+        "#{services}"
+      end
+      def master_managed_services
+        "cloud_master_takeover #{services}"
+      end
+      def launching_user_data
+        {:polling_time => polling_time}.to_yaml
+      end
+      # Keypair path
+      # Idiom:
+      #  /Users/username/.ec2/id_rsa-name
+      def keypair_path
+        "#{ec2_dir}/id_rsa#{keypair ? "-#{keypair}" : "" }"
+      end
+      # Are we in development or test mode
+      %w(development production test).each do |env|
+        eval <<-EOE
+          def #{env}?
+            environment == '#{env}'
+          end
+        EOE
+      end
+      def environment=(env)
+        environment = env
+      end
+      def maintain_pid_path
+        "/var/run/pool_maintain.pid"
+      end
+      %w(scp_instances_script reconfigure_instances_script).each do |file|
+        define_method "sh_#{file}" do
+          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.sh")
+        end
+      end
+      # Standard configuration files
+      %w(haproxy monit heartbeat heartbeat_authkeys).each do |file|
+        define_method "#{file}_config_file" do
+          File.join(File.dirname(__FILE__), "../..", "config", "#{file}.conf")
+        end
+      end
+      def version
+        PoolParty::Version::STRING
+      end
+      def install_on_load?
+        options.install_on_load == true
+      end
+      # Call the options from the Application
+      def method_missing(m,*args)
+        options.methods.include?("#{m}") ? options.send(m,args) : super
+      end
+    end
+        
+  end
+    
+end