auser-poolparty 0.2.26 → 0.2.35

data/lib/erlang/messenger/src/pm_node_supervisor.erl

@@ -24,17 +24,15 @@ start_link(Args) ->
 
 init([]) ->
 	RestartStrategy = one_for_one,
-	MaxRestarts = 3, % 1000
-	MaxTimeBetRestarts = 30, % 3600
+	MaxRestarts = 1000,
+	MaxTimeBetRestarts = 3600,
+	TimeoutTime = 5000,
 
 	SupFlags = {RestartStrategy, MaxRestarts, MaxTimeBetRestarts},
+	
+	EventManager = {pm_event_manager,  {pm_event_manager, start_link, []}, permanent, TimeoutTime, worker, dynamic},
+	NodeServer = {pm_node1, {pm_node, start_link, []}, permanent, TimeoutTime, worker, [pm_node]},
 
-	LoadServers = [
-		{pm_node1,
-			{pm_node, start_link, []}, 
-			permanent, 5000, worker, 
-			[pm_node]
-		}
-	],
+	LoadServers = [EventManager, NodeServer],
 
 	{ok, {SupFlags, LoadServers}}.

data/lib/erlang/messenger/src/utils.erl

@@ -35,4 +35,23 @@ average_for_list(Num, L) ->
 			0;
 		_ ->
 			Num / length(L)
-	end.
+	end.
+
+
+
+% Provisioning utils
+distribute_modules_to(Modules, Nodes) ->
+	% transfer the modules to all the nodes
+	io:format("Sending ~p to ~p~n", [Modules, Nodes]),
+	lists:foreach(fun(Node) ->
+			transfer_modules(Node, Modules)
+	end, Nodes).
+
+% Transfer modules of code to this node
+transfer_modules(Node, Modules) ->
+	[transfer_module(Node, M) || M <- Modules].
+
+% Transfer one module to the Node
+transfer_module(Node, Module) ->
+	{_Module, Binary, Filename} = code:get_object_code(Module),
+	rpc:call(Node, code, load_binary, [Module, Filename, Binary]).

data/lib/erlang/messenger/useful_snippets

@@ -0,0 +1,6 @@
+% Snippets
+% lists:map(fun(No) -> net_adm:ping(No) end, pm_cluster:any_new_servers()).
+
+% Start test nodes, defining testing
+% erl -pa ./ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -setcookie poolparty -boot pm_node_rel-0.1 -sname node0
+% erl -pa ./ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -setcookie poolparty -boot pm_node_rel-0.1 -sname node1

data/lib/poolparty/base_packages/haproxy.rb

@@ -20,15 +20,15 @@ module PoolParty
           has_service(:name => "haproxy")
 
           # Tempalte variables
-          variable(:name => "name_haproxy", :value => "#{@parent.name}")
+          variable(:name => "name_haproxy", :value => "#{cloud.name}")
           
-          if @parent.provisioning?
-            variable(:name => "nodenames_haproxy", :value => "#{list_of_running_instances.map{|a| "#{a.send :name}" }.join("\t")}")
-            variable(:name => "node_ips_haproxy",  :value => "#{list_of_running_instances.map{|a| "#{a.send :ip}" }.join("\t")}")
-          else
+          # if cloud.provisioning?
+          #   variable(:name => "nodenames_haproxy", :value => "#{list_of_running_instances.map{|a| "#{a.send :name}" }.join("\t")}")
+          #   variable(:name => "node_ips_haproxy",  :value => "#{list_of_running_instances.map{|a| "#{a.send :ip}" }.join("\t")}")
+          # else
             variable(:name => "nodenames_haproxy", :value => "generate('/usr/bin/env', '/var/lib/gems/1.8/bin/server-list-active', '-c', 'name')")
             variable(:name => "node_ips_haproxy",  :value => "generate('/usr/bin/env', '/var/lib/gems/1.8/bin/server-list-active', '-c', 'ip')")
-          end
+          # end
           
           variable(:name => "ports_haproxy", :value => ([(self.respond_to?(:port) ? port : Base.port)].flatten))        
           variable(:name => "forwarding_port", :value => (respond_to?(:forwarding_port) ? forwarding_port : Base.forwarding_port))

data/lib/poolparty/base_packages/poolparty.rb

@@ -2,23 +2,7 @@ module PoolParty
   class Base
     plugin :poolparty do
       
-      def enable        
-        # These are all requirements on the master
-        execute_if("$hostname", "master") do
-          has_cron({:name => "maintain script ", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3"})
-          # TODO: Update this so it only runs when needed
-          has_exec(:name => ". /etc/profile && server-start-master")
-          # has_exec(:name => "download-activesupport", :cwd => Base.remote_storage_path) do            
-          #   command "wget http://rubyforge.org/frs/download.php/45627/activesupport-2.1.2.gem -O activesupport.gem"
-          # end
-          # has_exec(:name => "download-ParseTree", :cwd => Base.remote_storage_path) do            
-          #   command "wget http://rubyforge.org/frs/download.php/45600/ParseTree-3.0.1.gem -O ParseTree.gem"
-          # end
-          # has_exec(:name => "download-RubyInline", :cwd => Base.remote_storage_path) do            
-          #   command "wget http://rubyforge.org/frs/download.php/45683/RubyInline-3.8.1.gem -O RubyInline.gem"
-          # end          
-        end
-        
+      def enable                
         has_package(:name => "erlang")
         has_package(:name => "erlang-dev")
         has_package(:name => "erlang-src")
@@ -27,20 +11,21 @@ module PoolParty
           # These should be installed automagically by poolparty, but just in case
           # TODO: Fix the requires method with a helper          
           g.has_gempackage(:name => "logging", :download_url => "http://rubyforge.org/frs/download.php/44731/logging-0.9.4.gem")
+          g.has_gempackage(:name => "hoe", :download_url => "http://rubyforge.org/frs/download.php/45685/hoe-1.8.2.gem")
+          g.has_gempackage(:name => "rake", :download_url => "http://rubyforge.org/frs/download.php/43954/rake-0.8.3.gem")
           g.has_gempackage(:name => "xml-simple") do |x|
-            x.has_gempackage(:name => "grempe-amazon-ec2", :source => "http://gems.github.com")
+            x.has_gempackage(:name => "grempe-amazon-ec2", :download_url => "http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem")
           end
           
           has_gempackage(:name => "ParseTree", :download_url => "http://rubyforge.org/frs/download.php/45600/ParseTree-3.0.1.gem") do |pt|
             pt.has_gempackage(:name => "ruby2ruby", :download_url => "http://rubyforge.org/frs/download.php/45587/ruby2ruby-1.2.0.gem")
             pt.has_gempackage(:name => "activesupport", :download_url => "http://rubyforge.org/frs/download.php/45627/activesupport-2.1.2.gem") do |a|
-              a.has_gempackage(:name => "auser-poolparty", :source => "http://gems.github.com") do |pool|
-                pool.has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger", :requires => get_gempackage("auser-poolparty")) do |mess|
-                  mess.has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node")
-                end                
-              end
+              a.has_gempackage(:name => "poolparty", :download_url => "http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty-latest.gem?raw=true")
             end
             has_gempackage(:name => "RubyInline", :download_url => "http://rubyforge.org/frs/download.php/45683/RubyInline-3.8.1.gem")
+            
+            has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger", :requires => get_gempackage("poolparty"))
+            has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node", :requires => get_exec("build_messenger"))
           end          
           
         end
@@ -53,29 +38,23 @@ module PoolParty
         
         # Custom run puppet to minimize footprint
         # TODO: Update the offsetted times
-        has_cron(:name => "puppetd runner", :user => Base.user, :minute => [0,15,30,45]) do
+        has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/8") do
           command((self.respond_to?(:master) ? self : parent).master.puppet_runner_command)
         end
-        # has_host(:name => "puppet", :ip => (self.respond_to?(:master) ? self : parent).master.ip)
-        custom_function <<-EOE
-define download_file(
-        $site="",
-        $cwd="",
-        $creates="",
-        $require="",
-        $user="") {                                                                                         
-
-    exec { $name:                                                                                                                     
-        command => "wget ${site}/${name}",                                                         
-        cwd => $cwd,
-        creates => "${cwd}/${name}",                                                              
-        require => $require,
-        user => $user,                                                                                                          
-    }
-
-}
         
-        EOE
+        # These are all requirements on the master
+        execute_if("$hostname", "master") do
+          has_cron({:name => "maintain script ", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3"})
+          # TODO: Update this so it only runs when needed
+          has_exec(:name => "start master messenger", :command => ". /etc/profile && server-start-master", :requires => [get_gempackage("poolparty"), get_exec("build_messenger")])
+          
+          has_remotefile(:name => "/usr/bin/puppetcleaner") do
+            mode 744
+            template File.join(File.dirname(__FILE__), "..", "templates/puppetcleaner")
+          end
+        end
+        
+        # has_host(:name => "puppet", :ip => (self.respond_to?(:master) ? self : parent).master.ip)
       end
       
     end  

data/lib/poolparty/core/object.rb

@@ -69,4 +69,7 @@ class Object
     # self.instance_eval &block if block
     meta_undef name
   end
+  def vputs(m="", o=self)
+    puts m if o.verbose
+  end
 end

data/lib/poolparty/helpers/console.rb

@@ -16,6 +16,10 @@ module PoolParty
       end
     end
     
+    def extract_cloud_from_options(o)
+      o.cloudname ? [cloud(o.cloudname.downcase.to_sym)] : [clouds[clouds.keys.first]]
+    end
+    
     # Clear all the pools and reload the console
     # Call within console to reset and reload the entire poolparty base
     # as well

data/lib/poolparty/helpers/messenger.rb

@@ -1,3 +1,6 @@
+=begin rdoc
+  The connection to the messenger from poolparty, the client
+=end
 module PoolParty
   module Messenger
     
@@ -5,16 +8,22 @@ module PoolParty
     def self.erl_command(hostname, extra="")
       command_line_opts = "-pa #{append_dir}/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname #{hostname} -setcookie poolparty"
       
-      "erl #{command_line_opts} #{extra} 2>&1 &"
+      "erl #{command_line_opts} #{extra} 2>&1"
     end
     
     def self.append_dir
-      ::File.join( ::File.dirname(__FILE__), "..", "..", "erlang/messenger" )
+      ::File.expand_path(::File.join( ::File.dirname(__FILE__), "..", "..", "erlang/messenger" ))
     end
     
-    def messenger_send!(cmd="", testing=false)
-      cmd = Messenger.erl_command("client", "-rsh ssh -noshell -run pm_client #{cmd} -s erlang halt")
-      testing ? cmd : Kernel.system(cmd)
+    def self.messenger_send!(cmd="", testing=false)
+      command = Messenger.erl_command("client", "-rsh ssh -noshell -run pm_client #{cmd} -s erlang halt")
+      testing ? command : %x[#{command}]
+    end
+    
+    # Helper methods
+    def self.startup_remote_messenger(hostname, testing=false)
+      messenger_send!("")
+      testing ? command : %x[#{command}]
     end
     
   end

data/lib/poolparty/helpers/optioner.rb

@@ -42,7 +42,7 @@ module PoolParty
       opts.on('-V', '--version', 'Display the version')    { output_version ; exit 0 }
       opts.on('-v', '--verbose', 'Be verbose')    { @options[:verbose] = true }  
       opts.on('-s [file]', '--spec-file [file]', 'Set the spec file')      { |file| self.spec file }
-      opts.on('-t', '--test', 'Testing mode')    { self.testing "true" }
+      opts.on('-t', '--test', 'Testing mode')    { self.testing true }
                   
       blk.call(opts, self) if blk
       

data/lib/poolparty/helpers/provisioner_base.rb

@@ -10,6 +10,7 @@ module PoolParty
     # Convenience method to clean 
     def self.provision_master(cloud, testing=false)
       Provisioner::Master.new(cloud).process_install!(testing)
+      process_clean_reconfigure_for!(cloud.master, cloud, testing)
     end
 
     def self.configure_master(cloud, testing=false)
@@ -21,13 +22,14 @@ module PoolParty
     end
 
     def self.provision_slaves(cloud, testing=false)
-      cloud.nonmaster_nonterminated_instances.each do |sl|        
+      cloud.nonmaster_nonterminated_instances.each do |sl|
         provision_slave(sl, cloud, testing)
       end
     end
 
     def self.configure_slaves(cloud, testing=false)
       cloud.nonmaster_nonterminated_instances.each do |sl|
+        puts "Slave: #{sl.name} (#{sl.ip})"
         configure_slave(sl, cloud, testing)
       end
     end
@@ -40,6 +42,10 @@ module PoolParty
       Provisioner::Slave.new(instance, cloud).process_configure!(testing)
     end
     
+    def self.process_clean_reconfigure_for!(instance, cloud, testing=false)
+      Provisioner::Master.new(cloud).process_clean_reconfigure_for!(instance, testing)
+    end
+    
     class ProvisionerBase
       
       include Configurable
@@ -72,21 +78,37 @@ module PoolParty
       def name
         @instance.name
       end
+      # TODO: Clean up this method
       def process_install!(testing=false)
         error unless valid?
         write_install_file
         setup_runner(@cloud)
         
         unless testing
-          puts "Logging on to #{@instance.ip}" if verbose          
+          vputs "Logging on to #{@instance.ip} (#{@instance.name})"
           @cloud.rsync_storage_files_to(@instance)
-          @cloud.prepare_reconfiguration
+          vputs "Preparing configuration on the master"
+          process_clean_reconfigure_for!(@instance)
+          
+          before_install(@instance)
           
+          vputs "Logging in and running provisioning on #{@instance.name}"
           cmd = "cd #{Base.remote_storage_path} && chmod +x install_#{name}.sh && /bin/sh install_#{name}.sh && rm install_#{name}.sh"
-          hide_output do
-            @cloud.run_command_on(cmd, @instance)
-          end          
-        end        
+          verbose ? @cloud.run_command_on(cmd, @instance) : hide_output {@cloud.run_command_on(cmd, @instance)}
+          
+          process_clean_reconfigure_for!(@instance)
+          after_install(@instance)
+          
+        end
+        # We have to get the right generated data into the manifest
+        # TODO: Clean this setup
+        @cloud.provisioning_complete
+      end
+      # Install callbacks
+      # Before installation callback
+      def before_install(instance)        
+      end
+      def after_install(instance)        
       end
       def configure
         valid? ? configure_string : error
@@ -102,19 +124,30 @@ module PoolParty
         setup_runner(@cloud)
         
         unless testing
-          puts "Logging on to #{@instance.ip}" if verbose
+          vputs "Logging on to #{@instance.ip}"
           @cloud.rsync_storage_files_to(@instance)
+          process_clean_reconfigure_for!(@instance)
 
           cmd = "cd #{Base.remote_storage_path} && chmod +x configure_#{name}.sh && /bin/sh configure_#{name}.sh && rm configure_#{name}.sh"
-          @cloud.run_command_on(cmd, @instance)
+          verbose ? @cloud.run_command_on(cmd, @instance) : hide_output {@cloud.run_command_on(cmd, @instance)}
+          process_clean_reconfigure_for!(@instance)
         end
       end
-      def process_reconfigure!(testing=false)
+      def process_clean_reconfigure_for!(instance, testing=false)
+        vputs "Cleaning certs from master: #{instance.name}"
+        # puppetca --clean #{instance.name}.compute-1.internal; puppetca --clean #{instance.name}.ec2.internal
+        # find /etc/puppet/ssl -type f -exec rm {} \;
+        command = <<-EOE
+if [ -f '/usr/bin/puppetcleaner' ]; then /usr/bin/puppetcleaner; fi
+        EOE
+        @cloud.run_command_on(command, @cloud.master) unless testing
+      end
+      def process_reconfigure!(testing=false)        
         @cloud.run_command_on("puppetd --test  2>&1 &", @instance) unless testing
       end
-      def setup_runner(cloud)
-        cloud.prepare_to_configuration
-        cloud.build_and_store_new_config_file
+      def setup_runner(force=false)
+        @cloud.prepare_to_configuration
+        @cloud.build_and_store_new_config_file(force)
       end
       def valid?
         true
@@ -147,6 +180,7 @@ module PoolParty
       # These are run on all the provisioners, master or slave
       def default_install_tasks
         [
+          "#!/usr/bin/env sh",
           upgrade_system,
           fix_rubygems,
           install_puppet,
@@ -238,13 +272,18 @@ module PoolParty
       def upgrade_system
         case @os
         when :ubuntu
-          "          
+          "
+if grep -q 'http://mirrors.kernel.org/ubuntu hardy main universe' /etc/apt/sources.list
+then 
+echo 'Updated already'
+else
 touch /etc/apt/sources.list
 echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
 aptitude update -y <<heredoc
 Y
 heredoc
 aptitude autoclean
+fi
           "
         else
           "# No system upgrade needed"
@@ -254,7 +293,7 @@ aptitude autoclean
       def install_puppet
         "#{installer_for( puppet_packages )}"
       end
-
+      
       def create_poolparty_manifest
         <<-EOS
           cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes

data/lib/poolparty/helpers/provisioners/master.rb

@@ -22,13 +22,14 @@ module PoolParty
           setup_basic_structure,
           setup_configs,        
           setup_fileserver,
-          setup_autosigning,          
+          setup_autosigning,
+          install_poolparty,
+          start_puppetmaster
         ] << configure_tasks
       end
 
       def configure_tasks
         [
-          # start_puppetmaster,
           create_local_node,
           move_templates,
           create_poolparty_manifest,
@@ -74,9 +75,6 @@ mkdir -p /etc/poolparty
       def setup_autosigning
         <<-EOS
 echo "*" > /etc/puppet/autosign.conf
-killall ruby
-rm -rf /etc/puppet/ssl/*
-puppetmasterd --verbose
         EOS
       end
 
@@ -115,19 +113,49 @@ cp #{Base.remote_storage_path}/#{Base.default_specfile_name} #{Base.base_config_
           "cp #{Base.remote_storage_path}/#{@cloud.full_keypair_name} #{@cloud.remote_keypair_path}"
         end
       end
+      
+      def install_poolparty
+        <<-EOE
+cd /var/poolparty
+wget http://rubyforge.org/frs/download.php/44731/logging-0.9.4.gem -O logging.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45581/ZenTest-3.11.0.gem -O zentest.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45600/ParseTree-3.0.1.gem -O ParseTree.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45587/ruby2ruby-1.2.0.gem -O ruby2ruby.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45627/activesupport-2.1.2.gem -O activesupport.gem 2>&1
+wget http://rubyforge.org/frs/download.php/18366/xml-simple-1.0.11.gem -O xml-simple.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45683/RubyInline-3.8.1.gem -O RubyInline.gem 2>&1
+wget http://rubyforge.org/frs/download.php/42580/flexmock-0.8.3.gem -O flexmock.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45685/hoe-1.8.2.gem -O hoe.gem 2>&1
+wget http://rubyforge.org/frs/download.php/18698/lockfile-1.4.3.gem -O lockfile.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45546/rubyforge-1.0.1.gem -O rubyforge.gem 2>&1
+wget http://rubyforge.org/frs/download.php/43954/rake-0.8.3.gem -O rake.gem 2>&1
+wget http://rubyforge.org/frs/download.php/45589/sexp_processor-3.0.0.gem -O sexp_processor.gem 2>&1
+wget http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty-latest.gem?raw=true -O poolparty-latest.gem 2>&1
+wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1
+
+#{
+  %w(rake lockfile rubyforge hoe zentest sexp_processor flexmock logging activesupport RubyInline ParseTree ruby2ruby xml-simple poolparty-latest amazon-ec2).map do |dep|
+    "gem install -y --no-ri --no-rdoc #{dep}.gem\n"
+  end
+}
+
+# gem install -y --no-ri --no-rdoc  --source http://gems.github.com grempe-amazon-ec2
+# gem install -y --no-ri --no-rdoc  --source http://gems.github.com auser-poolparty
+        EOE
+      end
 
       # ps aux | grep puppetmasterd | awk '{print $2}' | xargs kill
       # rm -rf /etc/puppet/ssl
-      # puppetmasterd --verbose
       def start_puppetmaster
-        <<-EOS
+        <<-EOS        
+puppetmasterd --verbose
         EOS
       end
 
       # puppetd --listen --fqdn #{@instance.name}
       def restart_puppetd
         <<-EOS
-          . /etc/profile && #{@instance.puppet_runner_command}
+. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master 2>&1
         EOS
       end
     end