auser-poolparty 0.0.8

data/config/installers/ubuntu_install.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+
+apt-get clean && apt-get update
+
+echo 'increasing the memory for apt'
+echo 'APT::Cache-Limit "516777216";' >> /etc/apt/apt.conf.d/70debconf
+
+echo 'running ubuntu_install.sh'
+rm -rf /usr/local/src/*
+
+echo 'updating apt'
+apt-get update
+# Get the essentials
+echo 'building essentials'
+apt-get -y install build-essential
+
+echo 'Installing git'
+apt-get -y install git-core rsync
+
+# Install ruby
+echo 'Installing ruby...'
+apt-get -y install ruby1.8-dev ruby1.8 ri1.8 rdoc1.8 irb1.8 libreadline-ruby1.8 libruby1.8
+ln -sf /usr/bin/ruby1.8 /usr/local/bin/ruby
+ln -sf /usr/bin/ri1.8 /usr/local/bin/ri
+ln -sf /usr/bin/rdoc1.8 /usr/local/bin/rdoc
+ln -sf /usr/bin/irb1.8 /usr/local/bin/irb
+
+# Install rubygems
+echo '-- Installing Rubygems'
+# if [[ ! -f /usr/local/src/rubygems-1.1.1 ]]; then
+  cd /usr/local/src
+  wget http://rubyforge.org/frs/download.php/35283/rubygems-1.1.1.tgz
+  tar -xzf rubygems-1.1.1.tgz
+  rm rubygems-1.1.1.tgz
+  cd rubygems-1.1.1  
+  ruby setup.rb --no-rdoc --no-ri
+  ln -sf /usr/bin/gem1.8 /usr/bin/gem
+# fi
+
+# Install gems
+# if [[ which pool | grep -v "bin" ]]; then
+  gem update --system
+  gem install SQS aws-s3 amazon-ec2 aska rake poolparty --no-rdoc --no-ri --no-test
+# fi
+
+# Install haproxy
+# if [[ which haproxy | grep -v "bin" ]]; then
+  apt-get -y install haproxy
+  echo 'Configuring haproxy logging'
+  sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/haproxy
+  sed -i 's/SYSLOGD=\"\"/SYSLOGD=\"-r\"/g' /etc/default/syslogd
+  echo 'local0.* /var/log/haproxy.log' >> /etc/syslog.conf && /etc/init.d/sysklogd restart
+  /etc/init.d/haproxy restart
+# fi
+# Install heartbeat
+# if [[ which heartbeat | grep -v "bin" ]]; then
+  apt-get -y install heartbeat-2
+# fi
+
+# Install monit
+# if [[ which monit | grep -v "bin" ]]; then
+  apt-get -y install monit
+  sudo mkdir /etc/monit
+  sed -i 's/startup=0/startup=1/g' /etc/default/monit
+  /etc/init.d/monit start
+# fi
+
+# Install s3fuse
+# if [[ which s3fs | grep -v "bin" ]]; then
+  apt-get install -y libcurl4-openssl-dev libxml2-dev libfuse-dev
+  cd /usr/local/src && wget http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz 
+  tar -zxf s3fs-r166-source.tar.gz
+  cd s3fs/ && make
+  mv s3fs /usr/bin
+# fi  
+
+echo ' - installed from script!'

data/config/monit/haproxy.monit.conf

@@ -0,0 +1,7 @@
+check process haproxy with pidfile /var/run/haproxy.pid
+  start program = "/usr/sbin/haproxy -f /etc/haproxy.cfg -p /var/run/haproxy.pid"
+  stop program  = "/usr/bin/killall -9 haproxy"
+  if totalmem is greater than 100.0 MB for 4 cycles then restart
+  if cpu is greater than 50% for 2 cycles then alert
+  if cpu is greater than 80% for 3 cycles then restart
+  if loadavg(5min) greater than 10 for 8 cycles then restart

data/config/monit.conf

@@ -0,0 +1,9 @@
+# Configuration for Monit, a monitoring tool
+set daemon 20 # Perform a check every 20 seconds
+set logfile /var/log/monit.log
+
+set httpd port 2812 and
+    use address 0.0.0.0
+    allow localhost
+
+include /etc/monit.d/*

data/config/nginx.conf

@@ -0,0 +1,24 @@
+user  usr usr;
+worker_processes  2;
+
+http {
+    sendfile       on;
+    tcp_nopush     on;
+
+    keepalive_timeout  65;
+    tcp_nodelay        on;
+
+    upstream fast_mongrels  { server 127.0.0.1:4567; }
+
+    server {
+        listen       80;
+        server_name  srv;
+        root        /apps/pool-party;
+     
+     location / {
+         proxy_pass http://fast_mongrels;
+         break;
+     }
+      
+  }
+}

data/config/reconfigure_instances_script.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# Reconfigure master
+:config_master
+# Start this instance's master maintain script
+:start_pool_maintain
+# Make the ha.d/resource.d
+sudo mkdir /etc/ha.d/resource.d/
+# Set this hostname as appropriate in the cloud
+:set_hostname
+# Configure heartbeat
+sudo mkdir /etc/ha.d/resource.d/
+# Start heartbeat
+/etc/init.d/heartbeat start
+# Start s3fs
+:start_s3fs
+# Configure monit
+mkdir /etc/monit.d

data/config/sample-config.yml

@@ -0,0 +1,23 @@
+:app_name: "test_app"
+:user_id: "1619-6456-1164"
+:access_key: ""
+:secret_access_key: ""
+:ami: "ami-4f7a9f26"
+:size: small
+:polling_time: "30.seconds"
+:minimum_instances: 2
+:maximum_instances: 3
+:ec2_dir: "/Users/auser/.ec2"
+:keypair: auser
+:os: ubuntu
+:host_port: 80
+:client_port: 8001
+:shared_bucket: "pool-party-app-data"
+:services: nginx
+:environment: production
+:contract_when:
+  web > 10
+  cpu < 0.2
+:expand_when:
+  cpu > 0.45
+  web < 10

data/config/scp_instances_script.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+:cloud_master_takeover
+:config_file
+:authkeys
+:ha_d
+:haresources
+:resources
+:monitrc
+:monit_d
+:haproxy
+:hosts

data/lib/core/array.rb

@@ -0,0 +1,13 @@
+=begin rdoc
+  Array extensions
+=end
+require "enumerator"
+class Array
+  # Collection with the index
+  def collect_with_index &block
+    self.enum_for(:each_with_index).collect &block
+  end
+  def runnable
+    self.join(" \n ").runnable
+  end
+end

data/lib/core/exception.rb

@@ -0,0 +1,9 @@
+=begin rdoc
+  Exception overloads
+=end
+class Exception
+  # Gives us a nice_message for exceptions
+  def nice_message(padding="")
+    "#{padding}#{message}\n#{padding}" + backtrace.join("\n#{padding}")
+  end
+end

data/lib/core/float.rb

@@ -0,0 +1,13 @@
+class Float
+  def round_to(x)
+    (self * 10**x).round.to_f / 10**x
+  end
+
+  def ceil_to(x)
+    (self * 10**x).ceil.to_f / 10**x
+  end
+
+  def floor_to(x)
+    (self * 10**x).floor.to_f / 10**x
+  end
+end

data/lib/core/hash.rb

@@ -0,0 +1,11 @@
+=begin rdoc
+  Hash extentions
+=end
+class Hash
+  def safe_merge(other_hash)
+    merge(other_hash.delete_if {|k,v| has_key?(k) })
+  end
+  def safe_merge!(other_hash)
+    merge!(other_hash.delete_if {|k,v| has_key?(k) })
+  end
+end

data/lib/core/kernel.rb

@@ -0,0 +1,12 @@
+=begin rdoc
+  Kernel overloads
+=end
+module Kernel
+  # Nice wait instead of sleep
+  def wait(time=10)
+    sleep time.is_a?(String) ? eval(time) : time
+  end
+  def as(klass_or_obj, &block)
+    block.in_context(klass_or_obj).call
+  end
+end

data/lib/core/module.rb

@@ -0,0 +1,22 @@
+# Module overloads
+class Module
+  # Gives us alias_method_chain from rails
+  def alias_method_chain(target, feature)
+    aliased_target, punctuation = target.to_s.sub(/([?!=])$/, ''), $1
+    yield(aliased_target, punctuation) if block_given?
+
+    with_method, without_method = "#{aliased_target}_with_#{feature}#{punctuation}", "#{aliased_target}_without_#{feature}#{punctuation}"
+
+    alias_method without_method, target
+    alias_method target, with_method
+
+    case
+      when public_method_defined?(without_method)
+        public target
+      when protected_method_defined?(without_method)
+        protected target
+      when private_method_defined?(without_method)
+        private target
+    end
+  end
+end

data/lib/core/object.rb

@@ -0,0 +1,18 @@
+=begin rdoc
+  Basic, add an alias_method to the object class
+  Add returning to the object
+=end
+class Object
+  def alias_method(new_id, original_id)
+    original = self.method(original_id).to_proc
+    define_method(new_id){|*args| original.call(*args)}
+  end
+  def returning(receiver)
+    yield receiver
+    receiver
+  end
+  def extended(&block)
+    block.in_context(self).call
+    self
+  end
+end

data/lib/core/proc.rb

@@ -0,0 +1,15 @@
+class Proc
+  def bind(object)
+    block, time = self, Time.now
+    (class << object; self; end).class_eval do
+      method_name = "__bind_#{time.to_i}_#{time.usec}"
+      define_method(method_name, &block)
+      method = instance_method(method_name)
+      remove_method(method_name)
+      method
+    end.bind(object)
+  end
+  def in_context(klass_or_obj)
+    klass_or_obj.send(:eval, self.to_ruby)
+  end
+end

data/lib/core/string.rb

@@ -0,0 +1,49 @@
+class String
+  def hasherize(format=[])
+    hash = {}
+    i = 0
+    self.split(%r{[\n|\t|\s| ]+}).collect {|a| a.strip}.each do |f|
+      break unless format[i]
+      unless f == "" || f.nil?
+        hash[format[i]] = f
+        i+=1
+      end      
+    end
+    hash
+  end
+  def ^(h={})
+    self.gsub(/:([\w]+)/) {h[$1.to_sym] if h.include?($1.to_sym)}
+  end
+  def runnable
+    self.strip.gsub(/\n/, " && ")
+  end
+  def classify
+    self.capitalize
+  end
+  def bucket_objects
+    AWS::S3::Bucket.objects(self)
+  end
+  def bucket_object(key)
+    AWS::S3::S3Object.value key, self if bucket_object_exists?(key)
+  end
+  def bucket_object_exists?(key)
+    AWS::S3::S3Object.exists? key, self
+  end
+  def store_bucket_value(key, data)
+    AWS::S3::S3Object.store key, data, self unless bucket_object_exists?(key)
+  end
+  def delete_bucket_value(key)
+    AWS::S3::S3Object.delete(key, self) if bucket_object_exists?(key)
+  end
+  def bucket_exists?
+    begin
+      AWS::S3::Bucket.find(self)
+      return true
+    rescue
+      return false
+    end        
+  end
+  def delete_bucket
+    AWS::S3::Bucket.delete(self, :force => true) if bucket_exists?
+  end
+end

data/lib/core/time.rb

@@ -0,0 +1,41 @@
+=begin rdoc
+  Based off the rails Numeric class.
+  Gives us the ability to use nice phrases such as
+  30.seconds, 5.days, etc.
+=end
+class Numeric
+  def ago(time = Time.now)
+    time - self
+  end
+  alias :until :ago
+
+  def since(time = Time.now)
+    time + self
+  end
+  alias :from_now :since
+  
+  def seconds
+    self
+  end
+  alias :second :seconds
+
+  def minutes
+    self * 60
+  end
+  alias :minute :minutes  
+
+  def hours
+    self * 60.minutes
+  end
+  alias :hour :hours
+
+  def days
+    self * 24.hours
+  end
+  alias :day :days
+
+  def weeks
+    self * 7.days
+  end
+  alias :week :weeks
+end

data/lib/modules/callback.rb

@@ -0,0 +1,133 @@
+=begin rdoc
+  Basic callbacks
+=end
+module PoolParty
+  module Callbacks
+    module ClassMethods
+      def define_callback_module(mod)
+        callbacks << mod
+      end
+      def define_callback_class(cla)
+        classes << cla
+      end
+      def callback(type, m, *args, &block)
+        arr = []                
+        args.each do |arg|
+          arr << case arg.class.to_s
+          when "Hash"
+            arg.collect do |meth, klass|
+              case klass.class.to_s
+              when "String"
+                define_callback_class(klass)
+                "self.#{klass.to_s.downcase}.#{meth}(self)"
+              else
+                "#{klass}.send :#{meth}, self"
+              end              
+            end
+          when "Symbol"
+            "self.send :#{arg}, self"
+          end
+        end
+
+        string = ""
+        if block_given?
+          num = store_proc(block.to_proc)
+          arr << <<-EOM
+            self.class.get_proc(#{num}).bind(self).call
+          EOM
+        end
+
+        string = create_eval_for_mod_with_string_and_type!(m, type) do
+          arr.join("\n")
+        end        
+
+        mMode = Module.new {eval string}
+
+        define_callback_module(mMode)
+      end
+      def before(m, *args, &block)
+        callback(:before, m, *args, &block)
+      end
+      def after(m, *args, &block)
+        callback(:after, m, *args, &block)
+      end
+
+      def create_eval_for_mod_with_string_and_type!(meth, type=nil, &block)
+        str = ""
+        case type
+        when :before          
+          str << <<-EOD
+            def #{meth}(*args)
+              #{yield}
+              super
+            end
+          EOD
+        when :after
+          str << <<-EOD
+            def #{meth}(*args)
+              super
+              #{yield}
+            end
+          EOD
+        else
+          str << <<-EOD
+            def #{meth}(*args)
+              #{yield}
+            end
+          EOD
+        end
+        str
+      end
+
+      def callbacks
+        @callbacks ||= []
+      end
+      def classes
+        @classes ||= []
+      end
+    end
+
+    module InstanceMethods
+      def initialize(*args)
+        extend_callbacks
+        extend_callback_methods
+      end
+      
+      def extend_callback_methods
+        unless self.class.classes.empty? 
+          self.class.classes.each do |klass|
+            m = %{def #{klass.to_s.downcase};@#{klass.to_s.downcase} ||= #{klass}.new;end}
+            self.class.class_eval m unless self.class.method_defined?(m)
+          end
+        end
+      end
+      
+      def extend_callbacks
+        unless self.class.callbacks.empty?
+          self.class.callbacks.each do |mod|
+            self.extend(mod)
+          end
+        end
+      end
+    end
+
+    module ProcStoreMethods
+      def store_proc(proc)
+        proc_storage << proc
+        proc_storage.index(proc)
+      end
+      def get_proc(num)
+        proc_storage[num]
+      end
+      def proc_storage
+        @proc_store ||= []
+      end
+    end
+
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.extend         ProcStoreMethods
+      receiver.send :include, InstanceMethods
+    end    
+  end
+end

data/lib/modules/ec2_wrapper.rb

@@ -0,0 +1,82 @@
+module PoolParty
+  extend self
+  
+  module Ec2Wrapper
+    
+    module ClassMethods      
+    end
+    
+    module InstanceMethods
+      # Run a new instance, with the user_data and the ami described in the config
+      def launch_new_instance!
+        instance = ec2.run_instances(
+          :image_id => Application.ami, 
+          :user_data => "#{Application.launching_user_data}",
+          :minCount => 1,
+          :maxCount => 1,
+          :key_name => Application.keypair,
+          :size => "#{Application.size}")
+          
+        item = instance.RunInstancesResponse.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Shutdown the instance by instance_id
+      def terminate_instance!(instance_id)
+        ec2.terminate_instances(:instance_id => instance_id)
+      end
+      def associate_address_with(ip, instance_id)
+        ec2.associate_address(:instance_id => instance_id, :public_ip => ip)
+      end
+      # Instance description
+      def describe_instance(id)
+        instance = ec2.describe_instances(:instance_id => id)
+        item = instance.DescribeInstancesResponse.reservationSet.item.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      end
+      # Get instance by id
+      def get_instance_by_id(id)
+        get_instances_description.select {|a| a.instance_id == id}[0] rescue nil
+      end
+      # Get the s3 description for the response in a hash format
+      def get_instances_description
+        @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances)
+      end
+            
+      # EC2 connections
+      def ec2
+        @ec2 ||= EC2::Base.new(:access_key_id => Application.access_key, :secret_access_key => Application.secret_access_key)
+      end      
+    end
+    
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+  # Provides a simple class to wrap around the amazon responses
+  class EC2ResponseObject
+    def self.get_descriptions(resp)
+      rs = resp.DescribeInstancesResponse.reservationSet.item
+      rs = rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      out = begin
+        rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r.instancesSet.item)}.reject {|a| a.nil?  }
+      rescue Exception => e
+        begin
+          # Really weird bug with amazon's ec2 gem
+          rs.reject {|a| a.empty? }.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil?  }
+        rescue Exception => e
+          []
+        end                
+      end
+      out
+    end
+    def self.get_hash_from_response(resp)
+      {
+        :instance_id => resp.instanceId, 
+        :ip => resp.dnsName, 
+        :status => resp.instanceState.name, 
+        :launching_time => resp.launchTime
+      } rescue nil
+    end
+  end
+end

data/lib/modules/safe_instance.rb

@@ -0,0 +1,31 @@
+=begin rdoc
+  Make a command thread-safe
+=end
+require "monitor"
+module PoolParty
+  extend self
+  
+  module ThreadSafeInstance
+    
+    module ClassMethods
+      def make_safe(meth)
+       original_method = "_unsafe_#{meth}_"
+       alias_method original_method, meth
+       define_method(meth) {|*args| self.class.synchronize { self.send(original_method) } }
+       self
+      end  
+    end
+    
+    module InstanceMethods
+      def make_safe meth
+        self.class.make_safe meth
+      end
+    end
+    
+    def self.included(receiver)
+      receiver.extend         MonitorMixin
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end