auser-poolparty 0.2.26 → 0.2.35

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -19,13 +19,13 @@ module PoolParty
         <<-EOE
           if [ -z "$(grep -v '#' /etc/hosts | grep 'master')" ]; then echo "#{master_ip} puppet master" >> /etc/hosts; else echo "host already set"; fi
           cp #{Base.remote_storage_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
-          echo 'DAEMON_OPTS="-w 120 –fqdn #{@instance.name} –server master"' > /etc/default/puppet
+          echo 'DAEMON_OPTS="-w 120 --fqdn #{@instance.name} --server master"' > /etc/default/puppet
         EOE
       end
 
       def setup_configs
         <<-EOS          
-          echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf        
+          echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf
         EOS
       end
 
@@ -33,10 +33,9 @@ module PoolParty
       # puppetd --listen --fqdn #{@instance.name}
       def start_puppet
         <<-EOS
-          ps aux | grep "puppetmasterd" | awk '{print $2}' | xargs kill
-          rm -rf /etc/puppet/ssl*
-          puppetd --test  2>&1 &
-          rm -rf /etc/puppet/ssl*
+          /etc/init.d/puppetmaster start
+          /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master 2>&1
+          # rm -rf /etc/puppet/ssl/*
         EOS
       end
       

data/lib/poolparty/net/remote_bases/ec2.rb

@@ -33,27 +33,24 @@ module PoolParty
       describe_instances.select {|a| a[:name] == id}[0] rescue nil
     end
     def describe_instances
-      unless @describe_instances && !@describe_instances.empty?
-        @id = 0
-        @describe_instances = get_instances_description.each_with_index do |h,i|
-          if h[:status] == "running"
-            @name = @id == 0 ? "master" : "node#{@id}"
-            @id += 1
-          else
-            @name = "#{h[:status]}_node#{i}"
-          end
-          h.merge!({
-            :name => @name,
-            :hostname => h[:ip],
-            :ip => h[:ip].convert_from_ec2_to_ip
-          })
+      @id = 0
+      get_instances_description.each_with_index do |h,i|
+        if h[:status] == "running"
+          @name = @id == 0 ? "master" : "node#{@id}"
+          @id += 1
+        else
+          @name = "#{h[:status]}_node#{i}"
         end
-      end
-      @describe_instances
+        h.merge!({
+          :name => @name,
+          :hostname => h[:ip],
+          :ip => h[:ip].convert_from_ec2_to_ip
+        })
+      end      
     end
     # Get the s3 description for the response in a hash format
     def get_instances_description
-      @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances).sort_by {|a| a[:launching_time]}
+      EC2ResponseObject.get_descriptions(ec2.describe_instances).sort_by {|a| a[:launching_time]}
     end
     
     # Help create a keypair for the cloud
@@ -80,7 +77,8 @@ module PoolParty
         # "if [ -z $(grep -v '#' /etc/hosts | grep '#{o.name}') ]; then echo \"$(curl http://169.254.169.254/latest/meta-data/public-ipv4) #{o.name}\" >> /etc/hosts; fi",
         "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
         "hostname #{o.name}",
-        "echo #{o.name} > /etc/hostname"
+        "echo #{o.name} > /etc/hostname",
+        "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1 && gem install -y --no-ri --no-rdoc amazon-ec2.gem 2>&1"
       ]
     end
     

data/lib/poolparty/net/remote_instance.rb

@@ -64,8 +64,11 @@ module PoolParty
         "#{name}\t#{ip}"
       end
       
-      # Commands for the servers
       def puppet_runner_command
+        self.class.send :puppet_runner_command
+      end
+      # Commands for the servers
+      def self.puppet_runner_command
         ". /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master 2>&1"
       end
     end

data/lib/poolparty/net/remoter.rb

@@ -13,7 +13,8 @@ module PoolParty
         end
       end
       def run_command_on_command(cmd="ls -l", remote_instance=nil)
-        "#{ssh_command(remote_instance)} '#{cmd}'"
+        vputs "Running #{cmd} on #{remote_instance.name == %x[hostname].chomp ? "self (master)" : "#{remote_instance.name}"}"
+        remote_instance.name == %x[hostname].chomp ? %x[#{cmd}] : "#{ssh_command(remote_instance)} '#{cmd}'"
       end
       def ssh_command(remote_instance)
         "#{ssh_string} #{remote_instance.ip}"
@@ -30,7 +31,7 @@ module PoolParty
         ["-o StrictHostKeyChecking=no", "-l '#{Base.user}'", '-i "'+full_keypair_path+'"']
       end
       def rsync_command
-        "rsync --delete -azP --exclude cache -e '#{ssh_string}'"
+        "rsync -azP --exclude cache -e '#{ssh_string}'"
       end
       # Open the cached local copy of the instances list and 
       # create a new RemoteInstance from each line
@@ -129,6 +130,7 @@ module PoolParty
         if list_of_pending_instances.size == 0
           block.call if block
         else
+          vputs "Waiting for there to be no pending instances..."
           wait "5.seconds"
           when_no_pending_instances(&block)
         end
@@ -149,13 +151,16 @@ module PoolParty
       # and if the master is not running AND we can start a new instance
       # Then wait for the master to launch
       def launch_and_configure_master!(testing=false)
+        vputs "Requesting to launch new instance"
         request_launch_new_instances(1) if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running?
         
         when_no_pending_instances do
           wait "20.seconds"
+          vputs "Provisioning master..."
           hide_output { Provisioner.provision_master(self, testing) }
-        end
-        
+          PoolParty::Provisioner.reconfigure_master(self, !testing)
+          after_launched
+        end        
       end
       def is_master_running?
         !list_of_running_instances.select {|a| a.name == "master"}.first.nil?
@@ -185,10 +190,11 @@ module PoolParty
             @num_instances = nonmaster_nonterminated_instances.size
             last_instances = nonmaster_nonterminated_instances[(@num_instances - @num)..(@num_instances)]
             last_instances.each do |inst|
-              PoolParty::Provisioner.provision_slave(inst, self)
+              vputs "Provisioning #{inst.name} slave"
+              PoolParty::Provisioner.provision_slave(inst, self)              
             end
             PoolParty::Provisioner.reconfigure_master(self, force)
-            # prepare_reconfiguration
+            after_launched
           end
         end
       end
@@ -196,11 +202,24 @@ module PoolParty
       # If we can shutdown an instnace and the load allows us to contract
       # the cloud, then we should request_termination_of_non_master_instance
       def contract_cloud_if_necessary(force=false)
-        if can_shutdown_an_instance?          
+        if can_shutdown_an_instance?
+          before_shutdown
           request_termination_of_non_master_instance if should_contract_cloud?(force)
         end
       end
       
+      # Callbacks
+      
+      # After launch callback
+      # This is called after a new instance is launched
+      def after_launched(force=false)        
+      end
+      
+      # Before shutdown callback
+      # This is called before the cloud is contracted
+      def before_shutdown
+      end
+      
       # Rsync command to the instance
       def rsync_storage_files_to(instance=nil)
         hide_output do
@@ -229,13 +248,13 @@ module PoolParty
       end
       
       # Prepare reconfiguration on the master
-      # TODO: Fix the killall
       # TODO: Curious about the puppet/ssl problems...
       # puppetd --test --no-daemonize 2>&1 &
+      # rm -rf /etc/puppet/ssl/*; 
       def prepare_reconfiguration
         unless @prepared
-          cmd = "killall ruby 2>&1 && rm -rf /etc/puppet/ssl/*; puppetmasterd --verbose 2>&1"
-          run_command_on(cmd, master)
+          # cmd = "/etc/init.d/puppetmaster restart"
+          # run_command_on(cmd, master)
           @prepared = true
         end
       end

data/lib/poolparty/net/remoter_base.rb

@@ -77,7 +77,7 @@ module PoolParty
       # If no keypair is passed, select them all
       def list_of_instances(keyp=nil)
         key = keyp ? keyp : keypair
-        describe_instances.select {|a| key ? a[:keypair] == key : true } if describe_instances
+        @describe_instances ||= describe_instances.select {|a| key ? a[:keypair] == key : true } if describe_instances
       end
       # Instances
       # Get the master from the cloud
@@ -91,6 +91,7 @@ module PoolParty
       end
       # Reset the cache of descriptions
       def reset!
+        @describe_instances = nil
       end
       def self.included(other)
         PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)

data/lib/poolparty/plugins/git.rb

@@ -11,15 +11,14 @@ module PoolParty
         has_package(:name => "git-core")
         has_directory(:name => "#{cwd}")
         
-        exec({:name => "git-#{name}", :requires => package(:name => "git-core")}) do
+        exec({:name => "git-#{name}", :requires => get_package(:name => "git-core"), :requires => get_directory("#{cwd}")}) do          
           command parent.user ? "git clone #{parent.user}@#{parent.source} #{parent.path}" : "git clone #{parent.source} #{parent.to ? parent.to : ""}"
           cwd "#{parent.cwd if parent.cwd}"
           creates "#{::File.join( (parent.cwd ? parent.cwd : cwd), ::File.basename(parent.source, ::File.extname(parent.source)) )}/.git"
-
-          exec(:name => "update-#{name}") do
-            cwd ::File.dirname(parent.creates)
-            command "git pull"
-          end
+        end
+        exec(:name => "update-#{name}", :requires => get_exec("git-#{name}")) do
+          cwd get_exec("git-#{parent.name}").cwd
+          command "git pull"
         end
       end
       

data/lib/poolparty/pool/base.rb

@@ -14,8 +14,9 @@ module PoolParty
       :user => "root", # This should change here
       :base_keypair_path => "~/.ec2",
       :tmp_path => "/tmp/poolparty",
-      :remote_storage_path => "/var/poolparty",      
-      :fileserver_base => "puppet:///files",
+      :remote_storage_path => "/var/poolparty",
+      :remote_gem_path => "/var/poolparty/gems",
+      :fileserver_base => "puppet://master/files",
       :base_config_directory => "/etc/poolparty",
       :template_directory => "templates",
       :template_path => "/var/lib/puppet/templates",

data/lib/poolparty/pool/cloud.rb

@@ -3,7 +3,7 @@ require File.dirname(__FILE__) + "/resource"
 
 module PoolParty    
   module Cloud
-    def cloud(name=:main, &block)
+    def cloud(name=:app, &block)
       clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, self, &block))
     end
 
@@ -92,13 +92,14 @@ module PoolParty
       # TODO: Consider the benefits of moving all the manifest
       # classes to separate files and keeping the containing
       # references in the include
-      def build_and_store_new_config_file
-        @manifest = build_manifest
+      def build_and_store_new_config_file(force=false)
+        vputs "Building new manifest configuration file (forced: #{force})"
+        manifest = force ? rebuild_manifest : build_manifest
         config_file = ::File.join(Base.storage_directory, "poolparty.pp")
-        ::File.open(config_file, "w+") do |file|
-          file << "class poolparty {"
-          file << @manifest
-          file << "}"
+        ::File.open(config_file, "w") do |file|
+          # file << "class poolparty {"
+          file << manifest
+          # file << "}"
         end
       end      
       
@@ -110,16 +111,23 @@ module PoolParty
             
       # Configuration files
       def build_manifest
+        vputs "Building manifest"
         @build_manifest ||= build_from_existing_file
         unless @build_manifest
-          reset_resources!
+          
+          # reset_resources!
           add_poolparty_base_requirements
           
-          @build_manifest = build_short_manifest
+          @build_manifest = "class poolparty {\n #{build_short_manifest}\n}"
         end
         @build_manifest
       end
       
+      def rebuild_manifest
+        @build_manifest = nil
+        build_manifest
+      end
+      
       def build_short_manifest
         returning Array.new do |str|            
 
@@ -139,11 +147,7 @@ module PoolParty
       end
       
       def build_from_existing_file
-        if ::FileTest.file?("/etc/puppet/manifests/classes/poolparty.pp")
-          open("/etc/puppet/manifests/classes/poolparty.pp").read
-        else
-          nil
-        end
+        ::FileTest.file?("/etc/puppet/manifests/classes/poolparty.pp") ? open("/etc/puppet/manifests/classes/poolparty.pp").read : nil
       end
       
       # To allow the remote instances to do their job,
@@ -177,6 +181,10 @@ module PoolParty
       def provisioning_complete
         @provisioning_in_progress = false
       end
+      
+      def reset!
+        @build_manifest = @describe_instances = nil
+      end
             
       # Add to the services pool for the manifest listing
       def add_service(serv)

data/lib/poolparty/pool/plugin.rb

@@ -20,6 +20,15 @@ module PoolParty
       # Overwrite this method
       def enable
       end
+      
+      # Call the cloud from within the plugin
+      def cloud
+        @p = parent
+        while !@p.is_a?(PoolParty::Cloud)
+          @p = @p.parent
+        end
+        @p
+      end
             
       # def method_missing(m, *args, &block)
       #   if block_given?

data/lib/poolparty/pool/pool.rb

@@ -1,7 +1,7 @@
 module PoolParty
   module Pool
     
-    def pool(name=:main, &block)
+    def pool(name=:app, &block)
       pools.has_key?(name) ? pools[name] : (pools[name] = Pool.new(name, &block))
     end    
     
@@ -15,7 +15,7 @@ module PoolParty
     end
         
     def reset!
-      $pools = $clouds = $plugins = nil
+      $pools = $clouds = $plugins = @describe_instances = nil
     end
 
     class Pool

data/lib/poolparty/pool/resources/class_package.rb

@@ -5,7 +5,7 @@ module PoolParty
     def classpackage_with_self(parent=self, &block)
       @cp = PoolParty::Resources::Classpackage.new(parent.options, parent, &block)
       @cp.instance_eval {@resources = parent.resources}
-      parent.instance_eval {@resources = nil}
+      # parent.instance_eval {@resources = nil}
       @cp
     end
                 

data/lib/poolparty/pool/resources/gem_package.rb

@@ -10,13 +10,15 @@ module PoolParty
       # TODO: Add it so that it tries to pull the gem off the master fileserver first...
       def loaded(opts={}, parent=self)
         if download_url
-          execute_if("$hostname", "master") do
-            has_exec(:name => "download-#{name}", :cwd => Base.remote_storage_path, :command => "wget #{download_url} -O #{name}.gem", :ifnot => "test -f #{Base.remote_storage_path}/#{name}.gem")
-          end
+          
           has_file(:name => "#{Base.remote_storage_path}/#{name}.gem", :source => "#{Base.fileserver_base}/#{name}.gem")
           
+          execute_if("$hostname", "master") do
+            has_exec(:name => "download-#{name}", :cwd => Base.remote_storage_path, :command => "wget #{download_url} -O #{name}.gem", :ifnot => "test -f #{Base.remote_storage_path}/#{name}.gem", :notify => get_file("#{Base.remote_storage_path}/#{name}.gem"))
+          end          
+          
           has_exec(opts.merge({:name => "#{name}", :cwd =>"#{Base.remote_storage_path}", :path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin"})) do
-            command "gem install -y --no-ri --no-rdoc #{name}.gem"
+            command "gem install -y --no-ri --no-rdoc #{Base.remote_storage_path}/#{name}.gem"
             ifnot "gem list --local #{name} | grep #{name} #{"| grep #{version}" if version}"
             requires get_file("#{Base.remote_storage_path}/#{name}.gem")
           end

data/lib/poolparty/templates/puppetcleaner

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+. /etc/profile
+ps aux | grep puppetmaster | awk '{print $2}' | xargs kill
+rm -rf /etc/puppet/ssl
+/usr/bin/env puppetmasterd --verbose

data/lib/poolparty/version.rb

@@ -2,7 +2,7 @@ module PoolParty
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 26
+    TINY  = 35
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/poolparty.gemspec

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: poolparty
 version: !ruby/object:Gem::Version 
-  version: 0.2.26
+  version: 0.2.35
 platform: ruby
 authors: 
 - Ari Lerner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-10-27 00:00:00 -07:00
+date: 2008-10-30 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -50,7 +50,7 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        version: 1.7.0
+        version: 1.8.2
     version: 
 description: Self-healing, auto-scaling cloud computing tool
 email: 
@@ -60,6 +60,7 @@ executables:
 - cloud-add-keypair
 - cloud-configure
 - cloud-contract
+- cloud-ensure-provisioning
 - cloud-expand
 - cloud-list
 - cloud-maintain
@@ -70,6 +71,7 @@ executables:
 - cloud-ssh
 - cloud-start
 - cloud-terminate
+- messenger-get-load
 - pool
 - pool-console
 - pool-describe
@@ -106,6 +108,7 @@ files:
 - bin/cloud-add-keypair
 - bin/cloud-configure
 - bin/cloud-contract
+- bin/cloud-ensure-provisioning
 - bin/cloud-expand
 - bin/cloud-list
 - bin/cloud-maintain
@@ -116,6 +119,7 @@ files:
 - bin/cloud-ssh
 - bin/cloud-start
 - bin/cloud-terminate
+- bin/messenger-get-load
 - bin/pool
 - bin/pool-console
 - bin/pool-describe
@@ -145,6 +149,7 @@ files:
 - lib/erlang/messenger/README
 - lib/erlang/messenger/Rakefile
 - lib/erlang/messenger/control
+- lib/erlang/messenger/ebin/erl_crash.dump
 - lib/erlang/messenger/ebin/master.app
 - lib/erlang/messenger/ebin/master_app.beam
 - lib/erlang/messenger/ebin/node.app
@@ -152,8 +157,9 @@ files:
 - lib/erlang/messenger/ebin/packager.app
 - lib/erlang/messenger/ebin/pm_client.beam
 - lib/erlang/messenger/ebin/pm_cluster.beam
-- lib/erlang/messenger/ebin/pm_event_handler.beam
+- lib/erlang/messenger/ebin/pm_event_manager.beam
 - lib/erlang/messenger/ebin/pm_master.beam
+- lib/erlang/messenger/ebin/pm_master_event_handler.beam
 - lib/erlang/messenger/ebin/pm_master_rel-0.1.rel
 - lib/erlang/messenger/ebin/pm_master_supervisor.beam
 - lib/erlang/messenger/ebin/pm_node.beam
@@ -162,6 +168,7 @@ files:
 - lib/erlang/messenger/ebin/pm_packager.beam
 - lib/erlang/messenger/ebin/utils.beam
 - lib/erlang/messenger/erl_crash.dump
+- lib/erlang/messenger/include/defines.hrl
 - lib/erlang/messenger/lib/eunit/AUTHORS
 - lib/erlang/messenger/lib/eunit/CHANGELOG
 - lib/erlang/messenger/lib/eunit/COPYING
@@ -217,21 +224,26 @@ files:
 - lib/erlang/messenger/lib/eunit/src/file_monitor.erl
 - lib/erlang/messenger/lib/eunit/sys.config
 - lib/erlang/messenger/lib/eunit/vsn.mk
+- lib/erlang/messenger/pm_master.beam
 - lib/erlang/messenger/pm_master_rel-0.1.boot
 - lib/erlang/messenger/pm_master_rel-0.1.script
+- lib/erlang/messenger/pm_node.beam
 - lib/erlang/messenger/pm_node_rel-0.1.boot
 - lib/erlang/messenger/pm_node_rel-0.1.script
 - lib/erlang/messenger/src/master_app.erl
 - lib/erlang/messenger/src/node_app.erl
 - lib/erlang/messenger/src/pm_client.erl
 - lib/erlang/messenger/src/pm_cluster.erl
-- lib/erlang/messenger/src/pm_event_handler.erl
+- lib/erlang/messenger/src/pm_event_manager.erl
 - lib/erlang/messenger/src/pm_master.erl
+- lib/erlang/messenger/src/pm_master_event_handler.erl
 - lib/erlang/messenger/src/pm_master_supervisor.erl
 - lib/erlang/messenger/src/pm_node.erl
 - lib/erlang/messenger/src/pm_node_supervisor.erl
 - lib/erlang/messenger/src/pm_packager.erl
 - lib/erlang/messenger/src/utils.erl
+- lib/erlang/messenger/useful_snippets
+- lib/erlang/messenger/utils.beam
 - lib/poolparty.rb
 - lib/poolparty/base_packages/haproxy.rb
 - lib/poolparty/base_packages/heartbeat.rb
@@ -328,6 +340,7 @@ files:
 - lib/poolparty/templates/namespaceauth.conf
 - lib/poolparty/templates/poolparty.monitor
 - lib/poolparty/templates/puppet.conf
+- lib/poolparty/templates/puppetcleaner
 - lib/poolparty/version.rb
 - lib/poolpartycl.rb
 - poolparty.gemspec
@@ -432,7 +445,8 @@ post_install_message: |-
   Please check out the documentation for any questions or check out the google groups at
     http://groups.google.com/group/poolpartyrb
   
-  Tutorials are at http://poolpartyrb.com
+  More tutorials can be found at 
+    http://poolpartyrb.com
   
   Don't forget to check out the plugin tutorial @ http://poolpartyrb.com to extend PoolParty for your needs!