auser-poolparty 0.0.8 → 0.0.9

data/lib/poolparty/optioner.rb

@@ -3,11 +3,15 @@ module PoolParty
     # Parse the command line options for options without a switch
     def self.parse(argv, safe=[])
       args = []
+      # Default options
+      safe.push "-v"
+      safe.push "-i"
+      
       argv.each_with_index do |arg,i|
-        unless arg.index("-")
-          args << arg 
-        else
-          argv.delete_at(i+1) unless safe.include?(arg)
+        unless arg.index("-") == 0# && !arg.match(/(?:[^"']+)/)
+          args << arg
+        else          
+          argv.shift unless safe.include?(arg)
         end
       end
       args

data/lib/poolparty/plugin.rb

@@ -12,14 +12,14 @@ module PoolParty
     #   create_methods :install, RemoteInstance
     # will give the following methods to the class
     #   before_install and after_install on the RemoteInstance
-    def self.create_methods(name, klass)
+    def self.create_methods(name, klass, opts={})
       str = ""
       %w(before after).each do |time|        
         str << <<-EOE
           def self.#{time}_#{name}(*meth)
             callee = self
             #{klass}.class_eval do
-              meth.each {|m| #{time} :#{name}, {m.to_sym => callee.to_s}}
+              meth.each {|m| #{time} :#{name}, {m.to_sym => callee.to_s }}
             end
           end
         EOE
@@ -27,10 +27,35 @@ module PoolParty
       eval str
     end
     
+    def self.user_tasks str
+      PoolParty::RemoteInstance.user_tasks << str
+    end
+    
+    def self.install_package_string(str)
+      PoolParty::Provider.define_user_package str
+    end
+    
+    def self.define_package_string(str)
+      PoolParty::Provider.define_user_install str
+    end
+    
+    def self.define_global_file(name, &block)
+      PoolParty::Master.define_global_user_file(name, &block)
+    end
+    
+    def self.define_node_file(name, &block)
+      PoolParty::Master.define_node_user_file(name, &block)
+    end
+    
+    def read_config_file(filename)
+      return {} unless filename
+      YAML.load(open(filename).read)
+    end
+    
     %w(install configure associate_public_ip become_master).each do |method|
       create_methods method, RemoteInstance
     end    
-    %w(start start_monitor scale_cloud reconfiguration add_instance terminate_instance check_stats).each do |method|
+    %w(start start_monitor install_cloud configure_cloud scale_cloud reconfiguration add_instance terminate_instance check_stats).each do |method|
       create_methods method, Master
     end    
     %w(define_tasks).each do |method|
@@ -38,6 +63,11 @@ module PoolParty
     end
     %w(run_tasks).each do |method|
       create_methods method, Scheduler
-    end    
+    end
+    
+    %w(load_packages).each do |method|
+      create_methods method, Provider
+    end
+    
   end  
 end

data/lib/poolparty/provider/packages/haproxy.rb

@@ -1,20 +1,5 @@
 # Install haproxy
-
-def post_install_string
-  %w(
-    "echo 'Configuring haproxy logging'"
-    "sed -i 's/ENABLED=0/ENABLED=1/g' /etc/default/haproxy"
-    "sed -i 's/SYSLOGD=\"\"/SYSLOGD=\"-r\"/g' /etc/default/syslogd"
-    "echo 'local0.* /var/log/haproxy.log' >> /etc/syslog.conf && /etc/init.d/sysklogd restart"
-    "/etc/init.d/haproxy restart"
-  )
-end
-
 package :haproxy, :provides => :proxy do
   description 'Haproxy proxy'
-  # version '1.2.18'
-  # source "http://haproxy.1wt.eu/download/1.2/src/haproxy-#{version}.tar.gz"
   apt %w( haproxy )
-  
-  post :install, post_install_string
 end

data/lib/poolparty/provider/packages/heartbeat.rb

@@ -1,4 +1,4 @@
-package :heartbeat do
+package :heartbeat, :provides => :failover do
   description "Heartbeat Linux HA project"
   apt %w(heartbeat-2)
 end

data/lib/poolparty/provider/packages/ruby.rb

@@ -2,7 +2,7 @@ package :ruby do
   description 'Ruby Virtual Machine'
   # version '1.8.6'
   # source "ftp://ftp.ruby-lang.org/pub/ruby/1.8/ruby-#{version}-p111.tar.gz" # implicit :style => :gnu
-  apt %w( ruby )
+  apt %w( ruby ruby1.8-dev )
   requires :ruby_dependencies
 end
 
@@ -18,19 +18,19 @@ package :rubygems do
   #   custom_install 'ruby setup.rb'
   # end
   apt %w( rubygems )
-  post :install, "gem update --system"
+  post :install, "gem update --system", "gem sources -a http://gems.github.com"
   requires :ruby
 end
 
 package :poolparty_required_gems do
   description "required gems"
-  gems %w(SQS aws-s3 amazon-ec2 aska rake)
+  gems %w( SQS aws-s3 amazon-ec2 aska rake rcov auser-poolparty )
 end
 
-package :poolparty do
+package :required_gems do
   description "Pool party gem"
-  gem "auser-pool-party" do
-    source 'http://gems.github.com'
+  gem "poolparty" do
+    source 'http://gems.github.com -y'
   end
   
   required :poolparty_required_gems

data/lib/poolparty/provider/packages/s3fuse.rb

@@ -1,11 +1,18 @@
 package :s3fs do
   description "S3 Fuse project"
-  source "http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz"
-  
+  source "http://s3fs.googlecode.com/files/s3fs-r166-source.tar.gz" do
+    custom_dir 's3fs'
+    custom_install "make"
+    
+    post :install, "mv s3fs /usr/bin"
+  end
+    
   requires :s3fs_deps
 end
 
 package :s3fs_deps do
   description "S3 Fuse project dependencies"
   apt %w( libcurl4-openssl-dev libxml2-dev libfuse-dev )
+  
+  requires :build_essential
 end

data/lib/poolparty/provider/provider.rb

@@ -1,36 +1,83 @@
-require "sprinkle"
-
 module PoolParty
   class Provider
+    class << self
+      attr_accessor :user_packages
+    end
     
     def self.install_poolparty(ips)
+      @ips = ips
       
-      $:.unshift( File.dirname(__FILE__) )
-      
-      load_str = []
-      
-      Dir["#{File.expand_path(File.dirname(__FILE__))}/packages/*"].each {|f| load_str << open(f).read}
+      load_str = load_packages
             
       script=<<-EOS
-        
-        #{load_str.join("\n")}
+        #{load_str.join("\n")}        
         
         policy :poolparty, :roles => :app do
           requires :git
           requires :ruby
+          requires :failover
+          requires :proxy
           requires :monit
           requires :s3fs
-          requires :rsync
-          requires :heartbeat
-          requires :poolparty
-        end        
+          requires :rsync          
+          requires :required_gems          
+        end
+        
+        #{install_from_sprinkle_string}
+      EOS
+      
+      PoolParty.message "Installing required poolparty paraphernalia"
+      Sprinkle::Script.sprinkle script
+    end
+    
+    def self.install_userpackages(ips)
+      @ips = ips
+      
+      script=<<-EOS
+        #{load_strings.join("\n")}
         
-        deployment do 
+        policy :userpackages, :roles => :app do
+          #{user_packages.join("\n")}
+        end        
+
+        #{install_from_sprinkle_string} 
+      EOS
+
+      PoolParty.message "Installing user defined packages"
+      Sprinkle::Script.sprinkle script# unless load_strings.empty?
+    end
+    
+    def self.define_user_package string
+      user_packages << string
+    end
+    
+    def self.define_user_install string
+      load_strings << string
+    end
+    
+    def self.user_packages
+      @@user_packages ||= []
+    end
+    
+    def self.load_strings
+      @@load_strings ||= []
+    end
+    
+    def self.load_packages
+      load_str = []
+      returning load_str do
+        Dir["#{File.expand_path(File.dirname(__FILE__))}/packages/*"].each {|f| load_str << open(f).read}
+      end
+    end
+    
+    def self.install_from_sprinkle_string
+      <<-EOS
+        deployment do
           delivery :vlad do 
             
-            set :ssh_cmd, '#{RemoteInstance.ssh_string}'
+            set :ssh_flags, "#{RemoteInstance.ssh_string.gsub(/ssh/, '')}"
             
-            #{string_roles_from_ips(ips)} 
+            #{string_roles_from_ips(@ips)}            
           end
           
           source do
@@ -39,17 +86,10 @@ module PoolParty
             builds   '/root/builds'
           end
             
-        end 
+        end
       EOS
-      
-      PoolParty.message "Installing required poolparty paraphernalia"      
-      install_from_sprinkle_string script
     end
-    
-    def self.install_from_sprinkle_string(str)
-      Sprinkle::Script.sprinkle str
-    end
-    
+        
     def self.string_roles_from_ips(ips)
       ips.collect do |ip|
         "role :app, '#{ip}'"

data/lib/poolparty/remote_instance.rb

@@ -5,13 +5,14 @@ module PoolParty
     # ############################
     include PoolParty
     include Callbacks
+    include FileWriter
     
-    attr_reader :ip, :instance_id, :name, :status, :launching_time, :stack_installed 
-    attr_accessor :name, :number, :scp_configure_file, :configure_file, :plugin_string
+    attr_reader :ip, :instance_id, :name, :status, :launching_time, :stack_installed, :keypair 
+    attr_accessor :name, :number, :scp_configure_file, :configure_file, :plugin_string, :keypair
     
     # CALLBACKS
-    after :install, :mark_installed    
-    
+    after :install, :mark_installed
+    after :configure, :associate_public_ip
     def initialize(obj={})
       super
       
@@ -21,6 +22,7 @@ module PoolParty
       @number = obj[:number] || 0 # Defaults to the master
       @status = obj[:status] || "running"
       @launching_time = obj[:launching_time] || Time.now
+      @keypair = obj[:keypair] || Application.keypair
     end
     
     # Host entry for this instance
@@ -65,81 +67,100 @@ module PoolParty
         ssh("monit #{cmd} all")
       end
     end
-    # Configure the server with the new, sexy shell script
-    # This compiles all the scp commands into a shell script and then executes it
-    # then it will compile a list of the commands to operate on the instance
-    # and execute it
-    # This is how the cloud reconfigures itself
-    def configure(caller=nil)
-      associate_public_ip
-      scp_basic_config_files
-
-      Master.with_nodes do |node|
-        # These are node-specific
-        PoolParty.message "configuring #{node.name}"
-        node.scp_specific_config_files
+    def configure_tasks
+      {
+        :move_hostfile => change_hostname,
+        :config_master => configure_master,
+        :move_config_file => move_config_file,
+        :set_hostname => change_hostname,
+        :mount_s3_drive => mount_s3_drive,
+        :update_plugins => update_plugin_string,
+        :configure_monit => configure_monit,
+        :configure_authkeys => configure_authkeys,
+        :configure_resource_d => configure_resource_d,
+        :configure_haproxy => setup_haproxy,
+        :configure_heartbeat => configure_heartbeat,
+        :user_tasks => user_tasks
+      }
+    end
+    def user_tasks
+      @@user_tasks ||= []
+    end
+    def move_config_file
+      <<-EOC
+        mv #{remote_base_tmp_dir}/config.yml ~/.config
+        mkdir -p ~/.ec2
+        mv #{remote_base_tmp_dir}/keypair ~/.ec2/id_rsa-#{Application.keypair}
+      EOC
+    end
+    def configure_heartbeat
+      <<-EOC
+        mv #{remote_base_tmp_dir}/ha.cf /etc/ha.d/ha.cf
+        /etc/init.d/heartbeat start
+      EOC
+    end    
+    def configure_authkeys
+      <<-EOC
+        mkdir -p /etc/ha.d
+        mv #{remote_base_tmp_dir}/authkeys /etc/ha.d/
+      EOC
+    end
+    
+    def configure_master
+      if master?
+        <<-EOC
+          pool maintain -c ~/.config -l ~/plugins
+        EOC
+      else
+        ""
       end
-      configure_basics_through_ssh
     end
     
-    def configure_basics_through_ssh
-      # -l #{Application.plugin_dir}
-      cmd=<<-EOC
-        #{update_plugin_string}
-        chmod 700 /etc/monit/monitrc
-        pool maintain -c ~/.config
-        hostname -v #{name}
-        /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
+    def configure_resource_d
+      <<-EOC
+        mkdir -p /etc/ha.d/resource.d
+        mv #{remote_base_tmp_dir}/cloud_master_takeover /etc/ha.d/resource.d
+        mv #{remote_base_tmp_dir}/resource.d/* /etc/ha.d/resource.d
       EOC
-      execute_tasks do
-        ssh(cmd.runnable)
-      end
     end
     
-    def scp_string(src,dest,opts={})
+    def configure_monit
+      <<-EOC
+        mv #{remote_base_tmp_dir}/monitrc /etc/monit/monitrc
+        mkdir -p /etc/monit.d/
+        mv #{remote_base_tmp_dir}/monit.d/* /etc/monit.d/
+        chown #{Application.username} /etc/monit/monitrc
+        chmod 700 /etc/monit/monitrc
+      EOC
     end
     
-    def scp_basic_config_files
-      cmd=<<-EOC
-        mkdir -p /etc/ha.d/resource.d
-        mkdir -p /etc/monit
-        mkdir -p /etc/monit.d
+    def change_hostname
+      <<-EOC
+        mv #{remote_base_tmp_dir}/#{name}-hosts /etc/hosts
+        hostname -v #{name}
       EOC
-      execute_tasks do
-        scp(Application.heartbeat_authkeys_config_file, "/etc/ha.d", :dir => "/etc/ha.d/resource.d")
-        scp(conf_file("cloud_master_takeover"), "/etc/ha.d/resource.d/cloud_master_takeover", :dir => "/etc/ha.d/resource.d/")
-
-        scp(Application.config_file, "~/.config") if Application.config_file
-        Dir["#{root_dir}/config/resource.d/*"].each do |file|
-          scp(file, "/etc/ha.d/resource.d/#{File.basename(file)}")
-        end
-        scp(Application.monit_config_file, "/etc/monit/monitrc", :dir => "/etc/monit")
-        Dir["#{root_dir}/config/monit.d/*"].each do |file|
-          scp(file, "/etc/monit.d/#{File.basename(file)}")
-        end
-        
-        `mkdir -p tmp/`
-        File.open("tmp/pool-party-haproxy.cfg", 'w') {|f| f.write(Master.build_haproxy_file) }
-        scp("tmp/pool-party-haproxy.cfg", "/etc/haproxy.cfg")
-      end
     end
-    def scp_specific_config_files
-      execute_tasks({:dont_set_hosts => true}) do
-        if Master.requires_heartbeat?
-          hafile = "tmp/#{name}-pool-party-ha.cf"
-          File.open(hafile, 'w') {|f| f.write(Master.build_heartbeat_config_file_for(self)) }
-          single_scp(hafile, "/etc/ha.d/ha.cf")
-
-          haresources_file = "tmp/#{name}-pool-party-haresources"
-          File.open(haresources_file, 'w') {|f| f.write(Master.build_heartbeat_resources_file_for(self)) }
-          single_scp(haresources_file, "/etc/ha/haresources", :dir => "/etc/ha")
-        end
-        hosts_file = "tmp/#{name}-pool-party-hosts"
-        File.open(hosts_file, 'w') {|f| f.write(Master.build_hosts_file_for(self)) }
-        single_scp(hosts_file, "/etc/hosts")
-      end
+    
+    def setup_haproxy
+      <<-EOS
+        mv #{remote_base_tmp_dir}/haproxy /etc/haproxy.cfg
+        sed -i "s/ENABLED=0/ENABLED=1/g" /etc/default/haproxy
+        sed -i 's/SYSLOGD=""/SYSLOGD="-r"/g' /etc/default/syslogd
+        echo "local0.* /var/log/haproxy.log" >> /etc/syslog.conf && /etc/init.d/sysklogd restart
+        /etc/init.d/haproxy restart
+      EOS
     end
     
+    def mount_s3_drive
+      if Application.shared_bucket.empty?
+        ""
+      else
+        <<-EOC
+          mkdir -p /data && /usr/bin/s3fs #{Application.shared_bucket} -o accessKeyId=#{Application.access_key} -o secretAccessKey=#{Application.secret_access_key} -o nonempty /data
+        EOC
+      end
+    end
+        
     # Installs with one commandline and an scp, rather than 10
     def install
       # unless stack_installed?
@@ -156,7 +177,7 @@ module PoolParty
       run_now "ls -l"
     end
     # Associate a public ip if it is set and this is the master
-    def associate_public_ip
+    def associate_public_ip(c)
       associate_address_with(Application.public_ip, @instance_id) if master? && Application.public_ip && !Application.public_ip.empty?
     end
     # Become the new master
@@ -164,19 +185,20 @@ module PoolParty
       @master = Master.new
       @number = 0
       @master.nodes[0] = self
+      @master.configure_cloud
       configure
     end
+    # Placeholder
+    def configure      
+    end
     def update_plugin_string
       reset!
       str = "mkdir -p #{Application.plugin_dir} && cd #{Application.plugin_dir}\n"
       installed_plugins.each do |plugin_source|
         str << "git clone #{plugin_source}\n"
       end
+      str
     end
-    def update_plugins(c)
-      ssh(c.update_plugin_string)
-    end
-    after :configure, :update_plugins
     # Is this the master and if not, is the master running?
     def is_not_master_and_master_is_not_running?
       !master? && !Master.is_master_responding?
@@ -190,7 +212,6 @@ module PoolParty
         File.join(PoolParty.root_dir, "config", name)
       end        
     end
-     
     # Description in the rake task
     def description
       case @status
@@ -203,7 +224,7 @@ module PoolParty
       end
     end
     def stack_installed?
-      @stack_installed ||= (run_now('echo ~/.installed') == "installed")
+      @stack_installed ||= false
     end
     def mark_installed(caller=nil)
       run_now("echo 'installed' > ~/.installed")