auser-poolparty 0.2.53 → 0.2.54

data/lib/erlang/messenger/useful_snippets

@@ -6,7 +6,7 @@
 % erl -pa ./ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -setcookie poolparty -boot pm_node_rel-0.1 -sname node1
 
 % Starting the client on the master
-% erl -pa /var/lib/gems/1.8/gems/poolparty-0.2.47/lib/erlang/messenger/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname client -setcookie poolparty -run pm_client start
+% erl -pa /var/lib/gems/1.8/gems/poolparty-0.2.54/lib/erlang/messenger/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname client -setcookie poolparty -run pm_client start
 
 % Get the load
 % erl -pa /var/lib/gems/1.8/gems/poolparty-0.2.46/lib/erlang/messenger/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname client -setcookie poolparty -run pm_client start

data/lib/poolparty.rb

@@ -1,13 +1,21 @@
 require 'rubygems'
 
 # Load required gems
+@required_software = Array.new
 %w(activesupport ftools logging ruby2ruby).each do |lib|
   begin
     require lib
   rescue Exception => e
+    @required_software << lib
     puts "Could not find library #{lib}: #{e}"
-  end
-  
+  end  
+end
+
+unless @required_software.empty?
+  puts "*****************************************"
+  puts "*           Error                       *"
+  puts "* Missing required software             *"
+  puts ""
 end
 
 # Use active supports auto load mechanism

data/lib/poolparty/base_packages/poolparty.rb

@@ -7,7 +7,7 @@ module PoolParty
         # TODO: COME BACK AND CLEAN THIS UP
         (self.respond_to?(:list_of_running_instances) ? self : parent).list_of_running_instances.each do |ri|
           has_host({:name => "#{ri.name}", :ip => ri.ip })
-        end
+        end        
         
         has_package(:name => "erlang")
         has_package(:name => "erlang-dev")
@@ -40,16 +40,17 @@ module PoolParty
           
           has_gempackage(:name => "poolparty", :download_url => "http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=true", :requires => [get_gempackage("ruby2ruby"), get_gempackage("RubyInline"), get_gempackage("ParseTree")])
           
-          has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger", :requires => get_gempackage("poolparty"), :ifnot => "ps aux | grep beam | grep node")
-          has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node", :requires => get_exec("build_messenger"), :ifnot => "ps aux | grep beam | grep node")
-          
+          # , :ifnot => "/bin/ps aux | /bin/grep -q pm_node"
+          has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger -v")
+          has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node")
+          # has_runit_service("pm_node", "pm_node", File.join(File.dirname(__FILE__), "..", "templates/messenger/node/"))
         end
         
         # execute_on_node do
-          has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
-            requires get_gempackage("poolparty")
-            command(PoolParty::Remote::RemoteInstance.puppet_rerun_commad)
-          end
+        has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
+          requires get_gempackage("poolparty")
+          command(PoolParty::Remote::RemoteInstance.puppet_rerun_commad)
+        end
         # end
         
         # Cloud panel setup
@@ -62,7 +63,7 @@ module PoolParty
         
         # Custom run puppet to minimize footprint
         # TODO: Update the offsetted times
-        execute_on_master do          
+        execute_on_master do
           has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
             requires get_gempackage("poolparty")
             command(PoolParty::Remote::RemoteInstance.puppet_master_rerun_command)
@@ -70,20 +71,26 @@ module PoolParty
           has_cron(:name => "Load handler", :user => Base.user, :minute => "*/4") do
             requires get_gempackage("poolparty")
             command(". /etc/profile && cloud-handle-load")
+          end          
+          has_cron(:name => "provisioning ensurer", :user => Base.user, :minute => "*/3") do
+            requires get_gempackage("poolparty")
+            command ". /etc/profile && cloud-ensure-provisioning"
           end
+          # has_runit_service("client_server", "pm_client", File.join(File.dirname(__FILE__), "..", "templates/messenger/client/"))
+          # has_runit_service("master_server", "pm_master", File.join(File.dirname(__FILE__), "..", "templates/messenger/master/"))
+          # TODO: Update this so it only runs when needed          
+          # has_customservice(:name => "start master server", :pattern => "/pm_master/", :bin => ". /etc/profile && server-start-master")
+          # has_customservice(:name => "start client server", :pattern => "/client_service/", :bin => ". /etc/profile && server-start-client")                    
+          has_exec(:name => "start master messenger", :command => ". /etc/profile && server-start-master") #, :ifnot => "/bin/ps aux | /bin/grep -q pm_master"
+          has_exec(:name => "start client server", :command => ". /etc/profile && server-start-client") #, :ifnot => "/bin/ps aux | /bin/grep -q client_server"
           
-          # TODO: Update this so it only runs when needed
-          has_exec(:name => "start master messenger", :command => ". /etc/profile && server-start-master", :requires => [get_gempackage("poolparty"), get_exec("build_messenger")], :ifnot => "ps aux | grep beam | grep master")
-          
-          has_exec(:name => "start client server", :command => ". /etc/profile && server-start-master", :requires => [get_gempackage("poolparty"), get_exec("build_messenger"), get_exec("start master messenger")], :ifnot => "ps aux | grep beam | grep client")
-          
-          has_cron({:name => "maintain script", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3", :requires => [get_gempackage("poolparty"), get_cron("puppetd runner"), get_cron("Load handler"), get_exec("start master messenger"), get_service("haproxy"), get_exec("start client server")]})
+          has_cron({:name => "maintain script", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3", :requires => [get_gempackage("poolparty"), get_cron("puppetd runner"), get_cron("Load handler"), get_service("haproxy")]})
           
           has_remotefile(:name => "/usr/bin/puppetcleaner") do
             mode 744
             template File.join(File.dirname(__FILE__), "..", "templates/puppetcleaner")
           end
-        end
+        end        
         
         has_remotefile(:name => "/usr/bin/puppetrerun") do
           mode 744

data/lib/poolparty/base_packages/runit.rb

@@ -0,0 +1,21 @@
+=begin rdoc
+  Runit beginning
+  NOT IMPLEMENTED YET
+=end
+module PoolParty
+  class Base
+    plugin :runit do
+      
+      def enable
+        unless enabled
+          has_file(:name => "inittab", :path => "/etc/inittab", :mode => 0644, :owner => "root", :group => "root")
+          has_package(:name => "runit", :ensures => "latest", :requires => get_file("inittab"))
+          has_exec(:name => "/sbin/start runsvdir", :cwd => "/var/service")
+          # has_remotefile(:name => "/etc/event.d/runsvdir", :notify => get_exec("/sbin/start runsvdir"))
+          enabled true
+        end          
+      end
+      
+    end
+  end
+end

data/lib/poolparty/helpers/binary.rb

@@ -15,7 +15,6 @@ module PoolParty
       end
       def get_existing_spec_location
         [                            
-            "#{Base.base_config_directory}/#{Base.default_specfile_name}",
             "#{Base.remote_storage_path}/#{Base.default_specfile_name}", 
             ENV["POOL_SPEC"],
             "#{Base.default_specfile_name}",

data/lib/poolparty/helpers/provisioner_base.rb

@@ -139,6 +139,10 @@ module PoolParty
         end
       end
       def process_clean_reconfigure_for!(instance, testing=false)
+        if instance.is_a?(String)
+          name = instance
+          instance = MyOpenStruct.new(:name => name)
+        end
         vputs "Cleaning certs from master: #{instance.name}"
         # puppetca --clean #{instance.name}.compute-1.internal; puppetca --clean #{instance.name}.ec2.internal
         # find /etc/puppet/ssl -type f -exec rm {} \;

data/lib/poolparty/helpers/provisioners/master.rb

@@ -34,8 +34,7 @@ module PoolParty
           create_local_node,
           move_templates,
           create_poolparty_manifest,
-          restart_puppetd,
-          start_puppetmaster
+          restart_puppetd
         ]
       end
       
@@ -121,15 +120,16 @@ wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-
       end
       
       # ps aux | grep puppetmasterd | awk '{print $2}' | xargs kill
-      # rm -rf /etc/puppet/ssl
+      # /etc/init.d/puppetmaster stop; rm -rf /etc/puppet/ssl; /etc/init.d/puppetmaster start
       def start_puppetmaster
         <<-EOS
 . /etc/profile
 # /etc/init.d/puppetmaster stop #{unix_hide_string}
-# ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string} # just in case
-# rm -rf /etc/puppet/ssl
-# # Start it back up
-/etc/init.d/puppetmaster start #{unix_hide_string}
+ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string} # just in case
+rm -rf /etc/puppet/ssl
+# Start it back up
+puppetmasterd --verbose
+# /etc/init.d/puppetmaster start #{unix_hide_string}
         EOS
       end
 
@@ -152,7 +152,7 @@ node "#{ri.name}" inherits default {}
       def move_templates
         <<-EOS
 mkdir -p #{Base.template_path}
-cp #{Base.remote_storage_path}/#{Base.template_directory}/* #{Base.template_path}
+cp -R #{Base.remote_storage_path}/#{Base.template_directory}/* #{Base.template_path}
         EOS
       end
       
@@ -163,6 +163,13 @@ cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolpa
         EOS
       end
       
+      def clean_master_certs
+        returning String.new do |s|
+          s << "puppetca --clean master.compute-1.internal 2>&1 > /dev/null;"
+          s << "puppetca --clean master.ec2.internal 2>&1 > /dev/null"
+        end
+      end
+      
       def restart_puppetd
         <<-EOS
 . /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master #{unix_hide_string}

data/lib/poolparty/modules/file_writer.rb

@@ -10,6 +10,13 @@ module PoolParty
       path = ::File.join( Base.tmp_path, Base.template_directory, ::File.basename(file) )
       FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
+    def copy_directory_into_template_storage_directory(dir)
+      path = make_template_directory(dir)
+      Dir["#{dir}/*"].each do |file|
+        FileUtils.cp file, path unless ::File.exists?(::File.join(path, ::File.basename(file)))
+      end
+      ::File.basename(path)
+    end
     def write_to_file_in_storage_directory(file, str, preceded="", &block)
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       write_to_file(path, str, &block)
@@ -45,9 +52,10 @@ module PoolParty
     def make_base_directory
       FileUtils.mkdir_p Base.storage_directory unless ::File.directory?(Base.storage_directory)
     end
-    def make_template_directory
-      path = ::File.join(Base.tmp_path, Base.template_directory)
+    def make_template_directory(dir=nil)
+      path = dir ? ::File.join(Base.tmp_path, Base.template_directory, ::File.basename(dir)) : ::File.join(Base.tmp_path, Base.template_directory)
       FileUtils.mkdir_p path unless ::File.directory?(path)
+      path
     end
     def clear_base_directory
       FileUtils::rm_rf "#{Base.storage_directory}"

data/lib/poolparty/net/messenger.rb

@@ -2,9 +2,6 @@
   The connection to the messenger from poolparty, the client
 =end
 module PoolParty
-  def reconfigure_cloud!
-    Messenger.messenger_send!("reconfigure_cloud")
-  end
   module Messenger
     def with_socket(testing=false, &block)
       host = testing ? "localhost" : (master.ip)
@@ -51,6 +48,15 @@ module PoolParty
   module Cloud
     class Cloud
       include PoolParty::Messenger
+      
+      def get_current_nodes
+        nodes = messenger_send!("get_current_nodes")
+        nodes.split(" ").map {|a| a.split(/@/)[-1] }
+      end
+      
+      def reconfigure_cloud!
+        messenger_send!("reconfigure_cloud")
+      end
     end
   end
 end

data/lib/poolparty/net/remoter.rb

@@ -178,16 +178,17 @@ module PoolParty
         reset!
         when_no_pending_instances do
           vputs "Waiting for 10 seconds"
-          wait "10.seconds" # Give some time for ssh to startup
+          wait "10.seconds" # Give some time for ssh to startup          
           @num_instances = list_of_running_instances.size
           vputs "(@num_instances - (num))..(@num_instances): #{(@num_instances - (num))..(@num_instances)}"
           last_instances = nonmaster_nonterminated_instances[(@num_instances - (num))..(@num_instances)]
           last_instances.each do |inst|
             vputs "Provision slave: #{inst}"
+            hide_output {PoolParty::Provisioner.process_clean_reconfigure_for!(inst, self)}
             PoolParty::Provisioner.provision_slave(inst, self, false) unless inst.master? rescue vputs "Error"
-            cmd = ". /etc/profile && cloud-provision -i #{inst.name.gsub(/node/, '')} #{unix_hide_string} &"
-            vputs "Provision slave with command #{cmd}"
-            Kernel.system cmd
+            hide_output {PoolParty::Provisioner.process_clean_reconfigure_for!(inst, self)}
+            # cmd = ". /etc/profile && cloud-provision -i #{inst.name.gsub(/node/, '')} &"
+            # vputs "Provision slave with command #{cmd}"
           end
           PoolParty::Provisioner.reconfigure_master(self)
         end
@@ -235,10 +236,13 @@ module PoolParty
         if can_start_a_new_instance? && should_expand_cloud?(force)
           vputs "Expanding the cloud based on load"
           @num = 1
-          request_launch_new_instances(@num)
-          
-          vputs "request_launch_new_instances: #{@num}"
-          provision_slaves_from_n(@num)
+          @num.times do |i|
+            list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")          
+            reset!
+            vputs "request_launch_new_instances: #{@num}"
+            provision_slaves_from_n(@num)
+            after_launched
+          end
         end
       end
       # Contract the cloud

data/lib/poolparty/plugins/runit.rb

@@ -0,0 +1,96 @@
+module PoolParty
+  class Runit
+    define_resource(:runitservice) do
+      def has_runit_service(name="runitservice", downif="", templatedir="")
+        path = copy_templates_from_templatedir(templatedir)
+        call_function <<-EOC
+runit_service {
+"#{name}":
+  directory => "/etc/sv",
+  downif => "/bin/ps aux | grep -v grep | grep -q #{downif}",
+  templatedir => "#{path}";  
+}        
+        EOC
+      end
+
+      def copy_templates_from_templatedir(dir=nil)
+        raise TemplateNotFound.new("template directory given") unless dir
+        raise TemplateNotFound.new("template directory cannot be found #{dir}") unless ::File.readable?(dir)
+        copy_directory_into_template_storage_directory(dir)
+      end
+
+      custom_function <<-EOF
+define runit_service ($directory = "/etc/sv", $downif = "/bin/false", $templatedir) {
+
+  file { "$directory-$name":
+    path => "$directory/$name",
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => Class["runit"]
+  }
+
+  file { "$directory/$name/log":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory-$name"]
+  }
+
+  file { "$directory/$name/log/main":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory/$name/log"]
+  }
+
+  file { "/etc/init.d/$name":
+    ensure => $lsbdistid ? {
+      'CentOS' => "/usr/local/bin/sv",
+      default => "/usr/bin/sv",
+    },
+    require => [ File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "/var/service/$name":
+    ensure => "$directory/$name",
+    require => [ File["$directory-$name"], File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "$directory/$name/log/run":
+    content => template("$templatedir/log-run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory/$name/log"],
+    notify => Service[$name]
+  }
+
+  file { "$directory/$name/run":
+    content => template("$templatedir/run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory-$name"],
+    notify => Service[$name]
+  }
+  
+  service { "$name":
+    hasrestart => true,
+    hasstatus => true,
+    require => File["/etc/init.d/$name"]
+  }
+
+  exec { "$name-down":
+    command => "/etc/init.d/$name down",
+    onlyif => $downif,
+    require => File["/etc/init.d/$name"]
+  }
+}      
+      EOF
+    end    
+  end
+end

data/lib/poolparty/pool/cloud.rb

@@ -170,7 +170,7 @@ module PoolParty
       # Also note that there is no block associated. This is because we have written
       # all that is necessary in a method called enable
       # which is called when there is no block
-      def add_poolparty_base_requirements        
+      def add_poolparty_base_requirements
         heartbeat
         haproxy
         ruby

data/lib/poolparty/pool/resource.rb

@@ -167,7 +167,8 @@ module PoolParty
         [
           :subscribe, :owner, :group, :path, :mode, :source, :notify, :subscribe, :check, :creates, :cwd, :command, :ensure,
           :require, :schedule, :range, :alias, :hour, :minute, :user, :month, :monthday, :name, :onlyif, :unless, :refreshonly,
-          :refresh, :content, :template, :ip, :repeat, :provider, :key, :device, :fstype, :remounts, :options, :atboot
+          :refresh, :content, :template, :ip, :repeat, :provider, :key, :device, :fstype, :remounts, :options, :atboot, :before,
+          :binary, :status, :start, :stop, :restart, :pattern
         ]
       end
       def key

data/lib/poolparty/pool/resources/custom_service.rb

@@ -0,0 +1,30 @@
+module PoolParty    
+  module Resources
+        
+    class Customservice < Resource
+      
+      default_options({
+        :ensure => "running",
+        :provider => "base"
+      })
+      
+      def bin(arg)
+        options.merge!(:binary => arg)
+        options.merge!(:start => arg)
+        options.merge!(:stop => arg.gsub(/start/, 'stop'))
+        options.merge!(:restart => "#{arg.gsub(/start/, 'stop')} && #{arg}")
+      end
+                  
+      def present
+        "running"
+      end
+      def absent
+        "stopping"
+      end
+      def class_type_name
+        "Service"
+      end
+    end
+    
+  end
+end