auser-poolparty 0.0.8

data/lib/poolparty/plugin.rb

@@ -0,0 +1,43 @@
+=begin rdoc
+  Allow for plugins based in callbacks
+  
+  A plugin should be able to hook into any method and run their command
+  either before or after the plugin.
+=end
+module PoolParty
+  class Plugin
+    
+    # Create a class-level method for the name on the class
+    # For instance:
+    #   create_methods :install, RemoteInstance
+    # will give the following methods to the class
+    #   before_install and after_install on the RemoteInstance
+    def self.create_methods(name, klass)
+      str = ""
+      %w(before after).each do |time|        
+        str << <<-EOE
+          def self.#{time}_#{name}(*meth)
+            callee = self
+            #{klass}.class_eval do
+              meth.each {|m| #{time} :#{name}, {m.to_sym => callee.to_s}}
+            end
+          end
+        EOE
+      end
+      eval str
+    end
+    
+    %w(install configure associate_public_ip become_master).each do |method|
+      create_methods method, RemoteInstance
+    end    
+    %w(start start_monitor scale_cloud reconfiguration add_instance terminate_instance check_stats).each do |method|
+      create_methods method, Master
+    end    
+    %w(define_tasks).each do |method|
+      create_methods method, Tasks
+    end
+    %w(run_tasks).each do |method|
+      create_methods method, Scheduler
+    end    
+  end  
+end

data/lib/poolparty/plugin_manager.rb

@@ -0,0 +1,67 @@
+=begin rdoc
+  A convenience method for working with plugins. 
+  
+  Sits on top of github.
+=end
+require "git"
+module PoolParty
+  def installed_plugins
+    @@installed_plugins ||= PluginManager.extract_git_repos_from_plugin_dirs.uniq
+  end
+  class PluginManager
+    include Callbacks
+            
+    def self.install_plugin(location)
+      unless File.directory?(plugin_directory(location))
+        begin
+          Git.clone(location, plugin_directory(location))      
+          reset!
+        rescue Exception => e
+          puts "There was an error"
+          puts e
+        end
+      else
+        puts "Plugin already installed"
+      end
+    end
+    
+    def self.remove_plugin(name)
+      Dir["#{PoolParty.root_dir}/#{PoolParty.plugin_dir}/*"].select {|a| a =~ /#{name}/}.each do |dir|
+        FileUtils.rm_rf dir
+      end
+    end
+    
+    def self.scan
+      returning Array.new do |a|
+        plugin_dirs.each do |plugin|
+          a << File.basename(plugin)
+        end
+      end
+    end
+    
+    def self.extract_git_repos_from_plugin_dirs
+      returning [] do |arr|
+        plugin_dirs.each do |dir|
+          begin
+            arr << open(File.join(dir, ".git", "config")).read[/url[\s]*=[\s](.*)/,1]
+          rescue Exception => e            
+          end          
+        end
+      end
+    end
+    
+    def self.plugin_dirs
+      Dir["#{PoolParty.user_dir}/vendor/*"]
+    end
+        
+    def self.plugin_directory(path)
+      File.join(base_plugin_dir, File.basename(path, File.extname(path)))
+    end
+    def self.create_plugin_directory
+      FileUtils.mkdir_p base_plugin_dir rescue ""
+    end
+    def self.base_plugin_dir
+      File.join(PoolParty.root_dir, "vendor")
+    end
+  end
+end

data/lib/poolparty/provider/packages/essential.rb

@@ -0,0 +1,6 @@
+## Special package, anything that defines a 'source' package means build-essential should be installed for Ubuntu
+
+package :build_essential do
+  description 'Build tools'
+  apt %w(build-essential)
+end

data/lib/poolparty/provider/packages/git.rb

@@ -0,0 +1,4 @@
+package :git do
+  description "Git SCM"
+  apt %w( zlib1g-dev git-core )
+end

data/lib/poolparty/provider/packages/haproxy.rb

@@ -0,0 +1,20 @@
+# Install haproxy
+
+def post_install_string
+  %w(
+    "echo 'Configuring haproxy logging'"
+    "sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/haproxy"
+    "sed -i 's/SYSLOGD=\"\"/SYSLOGD=\"-r\"/g' /etc/default/syslogd"
+    "echo 'local0.* /var/log/haproxy.log' >> /etc/syslog.conf && /etc/init.d/sysklogd restart"
+    "/etc/init.d/haproxy restart"
+  )
+end
+
+package :haproxy, :provides => :proxy do
+  description 'Haproxy proxy'
+  # version '1.2.18'
+  # source "http://haproxy.1wt.eu/download/1.2/src/haproxy-#{version}.tar.gz"
+  apt %w( haproxy )
+  
+  post :install, post_install_string
+end

data/lib/poolparty/provider/packages/heartbeat.rb

@@ -0,0 +1,4 @@
+package :heartbeat do
+  description "Heartbeat Linux HA project"
+  apt %w(heartbeat-2)
+end

data/lib/poolparty/provider/packages/monit.rb

@@ -0,0 +1,6 @@
+package :monit do
+  description "Monit monitoring service"
+  apt %w( monit )
+  
+  post :apt, "sudo mkdir /etc/monit", "sed -i 's/startup=0/startup=1/g' /etc/default/monit"
+end

data/lib/poolparty/provider/packages/rsync.rb

@@ -0,0 +1,4 @@
+package :rsync do
+  description "Rsync"
+  apt %w( rsync )
+end

data/lib/poolparty/provider/packages/ruby.rb

@@ -0,0 +1,37 @@
+package :ruby do
+  description 'Ruby Virtual Machine'
+  # version '1.8.6'
+  # source "ftp://ftp.ruby-lang.org/pub/ruby/1.8/ruby-#{version}-p111.tar.gz" # implicit :style => :gnu
+  apt %w( ruby )
+  requires :ruby_dependencies
+end
+
+package :ruby_dependencies do
+  description 'Ruby Virtual Machine Build Dependencies'
+  apt %w( bison zlib1g-dev libssl-dev libreadline5-dev libncurses5-dev file )
+end
+
+package :rubygems do
+  description 'Ruby Gems Package Management System'
+  # version '1.0.1'
+  # source "http://rubyforge.org/frs/download.php/29548/rubygems-#{version}.tgz" do
+  #   custom_install 'ruby setup.rb'
+  # end
+  apt %w( rubygems )
+  post :install, "gem update --system"
+  requires :ruby
+end
+
+package :poolparty_required_gems do
+  description "required gems"
+  gems %w(SQS aws-s3 amazon-ec2 aska rake)
+end
+
+package :poolparty do
+  description "Pool party gem"
+  gem "auser-pool-party" do
+    source 'http://gems.github.com'
+  end
+  
+  required :poolparty_required_gems
+end

data/lib/poolparty/provider/packages/s3fuse.rb

@@ -0,0 +1,11 @@
+package :s3fs do
+  description "S3 Fuse project"
+  source "http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz"
+  
+  requires :s3fs_deps
+end
+
+package :s3fs_deps do
+  description "S3 Fuse project dependencies"
+  apt %w( libcurl4-openssl-dev libxml2-dev libfuse-dev )
+end

data/lib/poolparty/provider/provider.rb

@@ -0,0 +1,60 @@
+require "sprinkle"
+
+module PoolParty
+  class Provider
+    
+    def self.install_poolparty(ips)
+      
+      $:.unshift( File.dirname(__FILE__) )
+      
+      load_str = []
+      
+      Dir["#{File.expand_path(File.dirname(__FILE__))}/packages/*"].each {|f| load_str << open(f).read}
+            
+      script=<<-EOS
+        
+        #{load_str.join("\n")}
+        
+        policy :poolparty, :roles => :app do
+          requires :git
+          requires :ruby
+          requires :monit
+          requires :s3fs
+          requires :rsync
+          requires :heartbeat
+          requires :poolparty
+        end        
+        
+        deployment do 
+          delivery :vlad do 
+            
+            set :ssh_cmd, '#{RemoteInstance.ssh_string}'
+            
+            #{string_roles_from_ips(ips)} 
+          end
+          
+          source do
+            prefix   '/usr/local'
+            archives '/root/sources'
+            builds   '/root/builds'
+          end
+            
+        end 
+      EOS
+      
+      PoolParty.message "Installing required poolparty paraphernalia"      
+      install_from_sprinkle_string script
+    end
+    
+    def self.install_from_sprinkle_string(str)
+      Sprinkle::Script.sprinkle str
+    end
+    
+    def self.string_roles_from_ips(ips)
+      ips.collect do |ip|
+        "role :app, '#{ip}'"
+      end.join("\n")
+    end
+    
+  end
+end

data/lib/poolparty/provider.rb

@@ -0,0 +1,2 @@
+$:.unshift(File.dirname(__FILE__))
+require "provider/provider"

data/lib/poolparty/remote_instance.rb

@@ -0,0 +1,216 @@
+module PoolParty
+  class RemoteInstance
+    # ############################
+    include Remoter
+    # ############################
+    include PoolParty
+    include Callbacks
+    
+    attr_reader :ip, :instance_id, :name, :status, :launching_time, :stack_installed 
+    attr_accessor :name, :number, :scp_configure_file, :configure_file, :plugin_string
+    
+    # CALLBACKS
+    after :install, :mark_installed    
+    
+    def initialize(obj={})
+      super
+      
+      @ip = obj[:ip]
+      @instance_id = obj[:instance_id]      
+      @name = obj[:name] || "node"
+      @number = obj[:number] || 0 # Defaults to the master
+      @status = obj[:status] || "running"
+      @launching_time = obj[:launching_time] || Time.now
+    end
+    
+    # Host entry for this instance
+    def hosts_entry
+      "#{@ip} #{name}"
+    end
+    # Internal host entry for this instance
+    def local_hosts_entry
+      "127.0.0.1 #{name}\n127.0.0.1 localhost.localdomain localhost ubuntu"
+    end
+    # Node entry for heartbeat
+    def node_entry
+      "node #{name}"
+    end    
+    # Internal naming scheme
+    def name
+      "#{@name}#{@number}"
+    end
+    # Entry for the heartbeat config file
+    def heartbeat_entry
+      "#{name} #{ip} #{Application.managed_services}"
+    end
+    # Entry for haproxy
+    def haproxy_entry
+      "\tserver #{name} #{@ip}:#{Application.client_port} weight 1 minconn 3 maxconn 6 check inter 20000 check"
+    end
+    def haproxy_resources_entry
+      "#{name} #{@ip}"
+    end
+    # Is this the master?
+    def master?
+      @number == 0
+    end
+    def secondary?
+      @number == 1
+    end
+    def set_hosts(c)
+    end
+    # Let's define some stuff for monit
+    %w(stop start restart).each do |cmd|
+      define_method "#{cmd}_with_monit" do
+        ssh("monit #{cmd} all")
+      end
+    end
+    # Configure the server with the new, sexy shell script
+    # This compiles all the scp commands into a shell script and then executes it
+    # then it will compile a list of the commands to operate on the instance
+    # and execute it
+    # This is how the cloud reconfigures itself
+    def configure(caller=nil)
+      associate_public_ip
+      scp_basic_config_files
+
+      Master.with_nodes do |node|
+        # These are node-specific
+        PoolParty.message "configuring #{node.name}"
+        node.scp_specific_config_files
+      end
+      configure_basics_through_ssh
+    end
+    
+    def configure_basics_through_ssh
+      # -l #{Application.plugin_dir}
+      cmd=<<-EOC
+        #{update_plugin_string}
+        chmod 700 /etc/monit/monitrc
+        pool maintain -c ~/.config
+        hostname -v #{name}
+        /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
+      EOC
+      execute_tasks do
+        ssh(cmd.runnable)
+      end
+    end
+    
+    def scp_string(src,dest,opts={})
+    end
+    
+    def scp_basic_config_files
+      cmd=<<-EOC
+        mkdir -p /etc/ha.d/resource.d
+        mkdir -p /etc/monit
+        mkdir -p /etc/monit.d
+      EOC
+      execute_tasks do
+        scp(Application.heartbeat_authkeys_config_file, "/etc/ha.d", :dir => "/etc/ha.d/resource.d")
+        scp(conf_file("cloud_master_takeover"), "/etc/ha.d/resource.d/cloud_master_takeover", :dir => "/etc/ha.d/resource.d/")
+
+        scp(Application.config_file, "~/.config") if Application.config_file
+        Dir["#{root_dir}/config/resource.d/*"].each do |file|
+          scp(file, "/etc/ha.d/resource.d/#{File.basename(file)}")
+        end
+        scp(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")
+        Dir["#{root_dir}/config/monit.d/*"].each do |file|
+          scp(file, "/etc/monit.d/#{File.basename(file)}")
+        end
+        
+        `mkdir -p tmp/`
+        File.open("tmp/pool-party-haproxy.cfg", 'w') {|f| f.write(Master.build_haproxy_file) }
+        scp("tmp/pool-party-haproxy.cfg", "/etc/haproxy.cfg")
+      end
+    end
+    def scp_specific_config_files
+      execute_tasks({:dont_set_hosts => true}) do
+        if Master.requires_heartbeat?
+          hafile = "tmp/#{name}-pool-party-ha.cf"
+          File.open(hafile, 'w') {|f| f.write(Master.build_heartbeat_config_file_for(self)) }
+          single_scp(hafile, "/etc/ha.d/ha.cf")
+
+          haresources_file = "tmp/#{name}-pool-party-haresources"
+          File.open(haresources_file, 'w') {|f| f.write(Master.build_heartbeat_resources_file_for(self)) }
+          single_scp(haresources_file, "/etc/ha/haresources", :dir => "/etc/ha")
+        end
+        hosts_file = "tmp/#{name}-pool-party-hosts"
+        File.open(hosts_file, 'w') {|f| f.write(Master.build_hosts_file_for(self)) }
+        single_scp(hosts_file, "/etc/hosts")
+      end
+    end
+    
+    # Installs with one commandline and an scp, rather than 10
+    def install
+      # unless stack_installed?
+      #   execute_tasks do
+      #     # scp(base_install_script, "~/base_install.sh")          
+      #     ssh("chmod +x base_install.sh && /bin/sh base_install.sh && rm base_install.sh && echo 'installed!' ")
+      #   end
+      #   PoolParty.message "After install execute_tasks"
+      #   mark_installed
+      # end
+    end
+    # Login to store the authenticity
+    def login_once
+      run_now "ls -l"
+    end
+    # Associate a public ip if it is set and this is the master
+    def associate_public_ip
+      associate_address_with(Application.public_ip, @instance_id) if master? && Application.public_ip && !Application.public_ip.empty?
+    end
+    # Become the new master
+    def become_master
+      @master = Master.new
+      @number = 0
+      @master.nodes[0] = self
+      configure
+    end
+    def update_plugin_string
+      reset!
+      str = "mkdir -p #{Application.plugin_dir} && cd #{Application.plugin_dir}\n"
+      installed_plugins.each do |plugin_source|
+        str << "git clone #{plugin_source}\n"
+      end
+    end
+    def update_plugins(c)
+      ssh(c.update_plugin_string)
+    end
+    after :configure, :update_plugins
+    # Is this the master and if not, is the master running?
+    def is_not_master_and_master_is_not_running?
+      !master? && !Master.is_master_responding?
+    end
+    # User conf file if it exists, or default one
+    def conf_file(name)
+      user_conf = File.join(PoolParty.user_dir, "config", name)
+      if File.file?(user_conf)
+        File.join(PoolParty.user_dir, "config", name)
+      else
+        File.join(PoolParty.root_dir, "config", name)
+      end        
+    end
+     
+    # Description in the rake task
+    def description
+      case @status
+      when "running"
+        "#{@number}: INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
+      when "shutting-down"
+        "(terminating) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
+      when "pending"
+        "(booting) INSTANCE: #{name} - #{@ip} - #{@instance_id} - #{@launching_time}"
+      end
+    end
+    def stack_installed?
+      @stack_installed ||= (run_now('echo ~/.installed') == "installed")
+    end
+    def mark_installed(caller=nil)
+      run_now("echo 'installed' > ~/.installed")
+      @stack_installed = true
+    end
+    def base_install_script
+      "#{root_dir}/config/installers/#{Application.os.downcase}_install.sh"
+    end
+  end
+end

data/lib/poolparty/remoter.rb

@@ -0,0 +1,106 @@
+=begin rdoc
+  Handle the remoting aspects of the remote_instances
+=end
+module PoolParty
+  module Remoter        
+    module ClassMethods
+      def ssh_string
+        "ssh -i #{Application.keypair_path} -o StrictHostKeyChecking=no -l #{Application.username}"
+      end
+    end
+    
+    module InstanceMethods      
+      include Callbacks
+      include Scheduler
+      
+      def scp local, remote, opts={}
+        data = open(local).read
+        begin
+          
+          cmd = "rsync --delete -azP -e '#{self.class.ssh_string} -l #{Application.username} -i #{Application.keypair_path}' "
+          
+          if opts[:dir]
+            ssh("mkdir -p #{opts[:dir]}")
+          end
+          
+          if opts[:single]
+            scp_tasks << "#{cmd} #{local} #{Application.username}@#{opts[:single]}:#{remote}"
+          else
+            target_hosts.each do |ip|
+              scp_tasks << "#{cmd} #{local} #{Application.username}@#{ip}:#{remote}"
+            end
+          end
+          
+        rescue Exception => e        
+        end      
+      end
+
+      def single_scp local, remote, opts={}
+        scp local, remote, opts.merge({:single => self.ip})
+      end
+
+      def ssh command="", opts={}, &block
+        cmd = self.class.ssh_string
+        if command.empty?
+          system("#{cmd} #{Application.username}@#{self.ip}")
+        else
+          if opts[:single]
+            ssh_tasks << "#{cmd} #{Application.username}@#{self.ip} '#{command.runnable}'"
+          else
+            target_hosts.each do |ip|
+              ssh_tasks << "#{cmd} #{Application.username}@#{ip} '#{command.runnable}'"
+            end
+          end
+        end
+      end
+      
+      def run_now command=""
+        system("#{self.class.ssh_string} #{Application.username}@#{self.ip} #{command.runnable}")
+      end
+      
+      def install *names
+        names.each {|name| install_tasks << name }
+      end
+      
+      def ssh_tasks;@ssh_tasks ||= [];end
+      def scp_tasks;@scp_tasks ||= [];end
+      def install_tasks;@install_tasks ||= [];end
+            
+      def reset!
+        @ssh_tasks = @scp_tasks = nil
+        @hosts = nil
+      end
+            
+      def execute_tasks(opts={})
+        # reset!
+        
+        set_hosts(nil) unless opts[:dont_set_hosts]
+        
+        yield if block_given?
+        
+        run_array_of_tasks(scp_tasks)
+        run_array_of_tasks(ssh_tasks)
+        
+        PoolParty.message "running #{ssh_tasks.size} tasks"
+      end
+      
+      def target_hosts
+        @hosts ||= Master.collect_nodes {|a| a.ip }
+      end
+      
+      def run_array_of_tasks(task_list)
+        add_task {`#{task_list.join(" && ")}`}
+        run_thread_list
+      end
+      
+      def set_hosts(c=nil)
+      end
+      
+    end
+  
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end