auser-poolparty 0.2.68 → 0.2.69

data/Manifest.txt

@@ -273,6 +273,7 @@ lib/poolparty/net/remote_instance.rb
 lib/poolparty/net/remoter.rb
 lib/poolparty/net/remoter_base.rb
 lib/poolparty/plugins/deploydirectory.rb
+lib/poolparty/plugins/git.rb
 lib/poolparty/plugins/line.rb
 lib/poolparty/plugins/rsyncmirror.rb
 lib/poolparty/plugins/runit.rb
@@ -323,6 +324,7 @@ lib/poolparty/templates/puppetrunner
 lib/poolparty/templates/yaws.conf
 lib/poolparty/version.rb
 lib/poolpartycl.rb
+lib/poolpartyspec.rb
 log/pool.logs
 poolparty.gemspec
 script/destroy
@@ -363,6 +365,7 @@ spec/poolparty/net/remote_spec.rb
 spec/poolparty/net/remoter_base_spec.rb
 spec/poolparty/net/remoter_spec.rb
 spec/poolparty/plugins/deploydirectory_spec.rb
+spec/poolparty/plugins/git_spec.rb
 spec/poolparty/plugins/line_spec.rb
 spec/poolparty/plugins/svn_spec.rb
 spec/poolparty/pool/base_spec.rb

data/PostInstall.txt

@@ -1,4 +1,4 @@
-Get ready to jump in the pool, you just installed PoolParty! (Updated at 11/19/08)
+Get ready to jump in the pool, you just installed PoolParty! (Updated at 22:49 11/24/08)
 
 To get started, run the generator:
 

data/README.txt

@@ -40,6 +40,9 @@ sudo gem install auser-poolparty
 
 == TODO:
 * Rewrite the disallowed_options to whitelist allowed_options on resources
+* Add pingback for nodes to master when failure occurs
+* Add become master for the nodes
+* Add soft-stop to haproxy while reconfiguring
 * Replace services with Runit
 * Refactor provisioning to use erlang
 * Add queuing of tasks on the messenger

data/Rakefile

@@ -23,12 +23,12 @@ task :build_local_gem => [:clean_tmp, :spec, :clean_pkg, :"manifest:refresh", :p
 desc "Packge with timestamp"
 task :update_timestamp do
   data = open("PostInstall.txt").read
-  str = "Updated at #{Time.now.strftime("%D")}"
+  str = "Updated at #{Time.now.strftime("%H:%M %D")}"
   
   if data.scan(/Updated at/).empty?
-    data = data.gsub(/just installed PoolParty\!/, '\0'+" (#{str})")
+    data = data ^ {:updated_at => str}    
   else
-    data = data ^ {:updated_at => str}
+    data = data.gsub(/just installed PoolParty\!(.*)$/, "just installed PoolParty! (#{str})")
   end
   ::File.open("PostInstall.txt", "w+") {|f| f << data }
 end

data/bin/cloud-configure

@@ -4,7 +4,8 @@ require "poolparty"
 require "poolpartycl"
 
 o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
-  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
+  optioner.cloudnames
+  # optioner.daemonizeable
 end
 
 o.loaded_clouds.each do |cloud|
@@ -13,7 +14,8 @@ o.loaded_clouds.each do |cloud|
     
     # hide_output do
       # if provision_class == "master" || provision_class == "all"
-        Provisioner.configure_master(self, testing)
+      @cl = self
+      daemon ? daemonize {Provisioner.configure_master(@cl, testing)} : Provisioner.configure_master(self, testing)
       # end
       # if provision_class == "slave" || provision_class == "all"
       #   Provisioner.configure_slaves(self, testing)

data/bin/cloud-start

@@ -4,8 +4,8 @@ require "poolparty"
 require "poolpartycl"
 
 o = PoolParty::Optioner.new(ARGV) do |opts, optioner|  
-  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
-  opts.on('-d', '--daemonize', 'Daemonize starting the cloud')    { optioner.daemon true }
+  optioner.cloudnames
+  optioner.daemonizeable
 end
 
 o.loaded_clouds.each do |cloud|
@@ -18,7 +18,8 @@ o.loaded_clouds.each do |cloud|
       #   puts("\tNot launching while in testing mode")
       # else
         vputs "Launching and configuring the master"
-        daemon ? daemonize {launch_and_configure_master!} : launch_and_configure_master!
+        @cl = self
+        daemon ? daemonize {@cl.launch_and_configure_master!} : launch_and_configure_master!
       # end
     # end
     when_no_pending_instances do

data/lib/poolparty.rb

@@ -81,8 +81,10 @@ class Class
 end
 
 ## Load PoolParty Plugins and package
-%w(plugins base_packages).each do |dir|
-  Dir[File.dirname(__FILE__) + "/poolparty/#{dir}/**.rb"].each do |file|
-    require file
+module PoolParty
+  %w(plugins base_packages).each do |dir|
+    Dir[::File.dirname(__FILE__) + "/poolparty/#{dir}/*.rb"].each do |file|
+      require file
+    end
   end
 end

data/lib/poolparty/helpers/binary.rb

@@ -47,7 +47,7 @@ module PoolParty
       def daemonize(&block)
         vputs "Daemonizing..."
         trap("CHLD") {Process.wait(-1, Process::WNOHANG)}        
-        fork do
+        pid = fork do
           Signal.trap('HUP', 'IGNORE') # Don't die upon logout
           File.open("/dev/null", "r+") do |devnull|
             $stdout.reopen(devnull)
@@ -56,6 +56,8 @@ module PoolParty
           end
           block.call if block
         end
+        Process.detach(pid)
+        pid
       end
       
     end

data/lib/poolparty/helpers/optioner.rb

@@ -11,17 +11,25 @@ module PoolParty
     include MethodMissingSugar
         
     def initialize(args=[], opts={}, &block)
-      @arguments = parse_args(args)
-      @parse_options = opts[:parse_options] ? opts[:parse_options] : true
+      @arguments = parse_args(args)      
       @extra_help = opts.has_key?(:extra_help) ? opts[:extra_help] : ""
-      @abstract = opts[:abstract] ? opts[:abstract] : false
-      @command = opts[:command] ? opts[:command] : false
+      @abstract = opts.has_key?(:abstract) ? opts[:abstract] : false
+      @load_pools = opts.has_key?(:load_pools) ? opts[:load_pools] : !@abstract
+      @parse_options = opts.has_key?(:parse_options) ? opts[:parse_options] : true
+      @command = opts.has_key?(:command) ? opts[:command] : false
       
       parse_options(&block) if @parse_options
       set_default_options
       self
     end
     
+    def daemonizeable
+      @opts.on('-d', '--daemonize', 'Daemonize starting the cloud')    { self.daemon true }
+    end
+    def cloudnames
+      @opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| self.cloudname c }
+    end
+    
     def parse_args(argv, safe=[])
       argv
     end
@@ -37,33 +45,34 @@ module PoolParty
     
     def parse_options(&blk)
       progname = $0.include?("-") ? "#{::File.basename($0[/(\w+)-/, 1])} #{::File.basename($0[/-(.*)/, 1])}" : ::File.basename($0)
-      opts = OptionParser.new 
-      opts.banner = "Usage: #{progname} #{@abstract ? "[command] " : ""}[options]"
+      @opts = OptionParser.new 
+      @opts.banner = "Usage: #{progname} #{@abstract ? "[command] " : ""}[options]"
 
-      opts.separator ""
+      @opts.separator ""
       
       unless @abstract
-        opts.separator "Options:"
+        @opts.separator "Options:"
         
-        opts.on('-v', '--verbose', 'Be verbose')    { self.verbose true }  
-        opts.on('-s [file]', '--spec-file [file]', 'Set the spec file')      { |file| self.spec file.chomp }
-        opts.on('-t', '--test', 'Testing mode')    { self.testing true }
-
-        blk.call(opts, self) if blk
+        @opts.on('-v', '--verbose', 'Be verbose')    { self.verbose true }  
+        @opts.on('-s [file]', '--spec-file [file]', 'Set the spec file')      { |file| self.spec file.chomp }
+        @opts.on('-t', '--test', 'Testing mode')    { self.testing true }
+        
+        blk.call(@opts, self) if blk
       end
       
-      opts.on('-V', '--version', 'Display the version')    { puts @version ; exit 0 }
-      opts.on_tail("-h", "--help", "Show this message") do
-        puts opts
+      @opts.on('-V', '--version', 'Display the version')    { puts @version ; exit 0 }
+      @opts.on_tail("-h", "--help", "Show this message") do
+        puts @opts
         puts @extra_help
         exit
       end
       
-      opts.parse(@arguments.dup)
+      @opts.parse(@arguments.dup)
       
       process_options
       output_options if verbose
-      unless @abstract
+      
+      if @load_pools
         self.loaded_pool load_pool(self.spec || Binary.get_existing_spec_location)
 
         self.loaded_clouds extract_cloud_from_options(self)

data/lib/poolparty/helpers/provisioner_base.rb

@@ -211,9 +211,10 @@ module PoolParty
         [
           "#!/usr/bin/env sh",
           upgrade_system,
-          fix_rubygems,
+          install_rubygems,
           make_logger_directory,
           install_puppet,
+          fix_rubygems,
           custom_install_tasks
         ] << install_tasks
       end
@@ -281,11 +282,19 @@ module PoolParty
         File.join(File.dirname(__FILE__), "..", "templates")
       end
       
-      def fix_rubygems
+      def install_rubygems
+        <<-EOE
+        #{installer_for("ruby rubygems")}
+        EOE
+      end
+      
+      def fix_rubygems        
         <<-EOE
-          #{installer_for("ruby rubygems")}
-          gem update --system # Force rubygems update
           echo '#{open(::File.join(template_directory, "gem")).read}' > /usr/bin/gem
+          echo 'Updating rubygems'
+          PAT=`/usr/bin/gem env gemdir`
+          /usr/bin/gem update --system #{unix_hide_string}
+          /usr/bin/gem update --system #{unix_hide_string}
         EOE
       end
 

data/lib/poolparty/helpers/provisioners/master.rb

@@ -122,7 +122,7 @@ wget http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=tru
 #{
   %w(rake lockfile rubyforge hoe ZenTest sexp_processor flexmock logging activesupport 
       RubyInline ParseTree ruby2ruby xml-simple poolparty).map do |dep|
-    "gem install --ignore-dependencies -y --no-ri --no-rdoc #{dep}.gem #{unix_hide_string}"
+    "/usr/bin/gem install --ignore-dependencies -y --no-ri --no-rdoc #{dep}.gem #{unix_hide_string}"
   end.join("\n")
 }
         EOE
@@ -186,7 +186,7 @@ cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolpa
           s << "puppetca --clean master.compute-1.internal 2>&1 > /dev/null"
           s << "puppetca --clean master.ec2.internal 2>&1 > /dev/null"
         end.join(";")
-        "if [ -f '/usr/bin/puppetcleaner' ]; then /usr/bin/env puppetcleaner; else #{str}; fi"
+        "if [ -f '/usr/bin/puppetrerun' ]; then /usr/bin/puppetrerun; else #{str}; fi"
       end
       
       def restart_puppetd

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -4,7 +4,7 @@ module PoolParty
   module CloudResourcer
     
     def plugin_directory(*args)
-      args = ["/plugins"] if args.empty?
+      args = ["#{::File.expand_path(Dir.pwd)}/plugins"] if args.empty?
       args.each {|arg| Dir["#{arg}/*/*.rb"].each {|f| require f rescue "" }}
     end
     

data/lib/poolparty/modules/file_writer.rb

@@ -6,7 +6,7 @@ module PoolParty
       FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
     def cleanup_storage_directory
-      Dir["#{Base.storage_directory}/**"].each do |f|
+      Dir["#{Base.storage_directory}/**/*"].each do |f|
         ::FileUtils.rm f if ::File.file?(f)
       end
     end
@@ -79,7 +79,7 @@ module PoolParty
       path
     end
     def clear_base_directory
-      Dir["#{Base.storage_directory}/**"].each do |f|
+      Dir["#{Base.storage_directory}/**/*"].each do |f|
         ::FileUtils.rm f if ::File.file?(f)
       end
     end

data/lib/poolparty/monitors/base_monitor.rb

@@ -1,7 +1,33 @@
 =begin rdoc
   Monitor class
   
-  TODO: Fill this out
+  Monitors are the basis for PoolParty scaling. Your cloud will expand and
+  contract against these monitors. You can set your cloud to be monitored by these
+  monitors simply by using them in the contract_when and the expand_when macros 
+  on your cloud like so:
+  
+    expand_when "cpu > 1.2", "memory > 0.94"
+    contract_when "cpu < 0.4", "memory < 0.3"
+  
+  You can also add your own monitors simply by creating a directory in the same
+  directory as the pool spec (the same directory as the plugin directory exists) and
+  placing your monitor file (format: [monitorname]_monitor.rb) there.
+  
+  Monitors are simply classes of the name of the monitor. They subclass the BaseMonitor
+  class from PoolParty. A sample monitor would look similar to:
+    
+    class SampleMonitor < PoolParty::Monitors::BaseMonitor
+      def run
+      end
+    end
+    register_monitor :sample
+  
+  The monitor class must have an instance level method called run. This method is called when 
+  the cloud is checking the monitor. The output of this method should be the output of
+  the monitor.
+  
+  Notice that at the end, you must call register_monitor :monitorname. This will tell your cloud
+  that it can monitor it with this monitor.
 =end
 require "#{::File.dirname(__FILE__)}/../pool/base"
 
@@ -49,7 +75,9 @@ module PoolParty
   end
 end
 
+# Require included monitors
 Dir["#{File.dirname(__FILE__)}/monitors/*.rb"].each {|f| require f}
+# Require custom monitors
 Dir["#{PoolParty::Base.custom_monitor_directories}/*.rb"].each {|f| require f}
 
 module PoolParty

data/lib/poolparty/net/messenger.rb

@@ -1,12 +1,15 @@
 =begin rdoc
-  The connection to the messenger from poolparty, the client
+  The connection to the messenger from poolparty, the client.
+  
+  This class will setup a socket connection to the master's client
+  at the messenger_client_port
 =end
 module PoolParty
   module Messenger
     def with_socket(testing=false, &block)
       host = testing ? "localhost" : (master.ip)
       vputs "Pinging #{host} with the messenger"
-      socket = TCPSocket.open(host, 7050)
+      socket = TCPSocket.open(host, Base.messenger_client_port)
       out = yield(socket)
       socket.close
       out

data/lib/poolparty/net/remote_bases/ec2.rb

@@ -1,3 +1,9 @@
+=begin rdoc
+  EC2 Remoter Base
+  
+  This serves as the basis for running PoolParty on Amazon's ec2 cloud
+  cluster. 
+=end
 require "date"
 require "#{::File.dirname(__FILE__)}/ec2/ec2_response_object"
 
@@ -65,18 +71,21 @@ begin
         EC2ResponseObject.get_descriptions(ec2.describe_instances)
       end
 
-      def after_launch_master(instance=nil)
+      def after_launch_master(inst=nil)
+        instance = master
+        vputs "Running tasks after launching the master"
         begin
-          when_no_pending_instances do
+          # when_no_pending_instances do
             if instance
-              ec2.associate_address(:instance_id => instance.instanceId, :public_ip => set_master_ip_to) if set_master_ip_to
-              ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instanceId, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
+              ec2.associate_address(:instance_id => instance.instance_id, :public_ip => set_master_ip_to) if set_master_ip_to
+              ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instance_id, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
             end
-          end
+          # end
         rescue Exception => e        
+          vputs "Error in after_launch_master: #{e}"
         end
         reset_remoter_base!
-        when_all_assigned_ips {wait "2.seconds"}
+        when_all_assigned_ips {wait "5.seconds"}
       end
 
       # Help create a keypair for the cloud
@@ -112,7 +121,8 @@ begin
           "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
           "hostname #{o.name}",
           "echo #{o.name} > /etc/hostname",
-          "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1 && gem install -y --no-ri --no-rdoc amazon-ec2.gem 2>&1"
+          "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
+          "/usr/bin/gem install -y --no-ri --no-rdoc amazon-ec2.gem 2>&1"
         ]
       end
 

data/lib/poolparty/net/remoter.rb

@@ -82,11 +82,15 @@ module PoolParty
         out
       end
       def request_launch_master_instance
-        inst = launch_new_instance!
+        @inst = launch_new_instance!
         wait "5.seconds"
-        when_no_pending_instances {after_launch_master(inst)}
+        when_no_pending_instances do
+          vputs "Master has launched"
+          reset!
+          after_launch_master(@inst)
+        end
       end
-      def after_launch_master(h={})
+      def after_launch_master(inst=nil)
         vputs "After launch master in remoter"
       end
       # Let's terminate an instance that is not the master instance

data/lib/poolparty/net/remoter_base.rb

@@ -1,3 +1,21 @@
+=begin rdoc
+  The base for Remote Bases
+  
+  By extending this class, you can easily add remoters to 
+  PoolParty. There are 4 methods that the remote base needs to implement
+  in order to be compatible.
+  
+  The four methods are:
+    launch_new_instance!
+    terminate_instance(id)
+    describe_instance(id)
+    describe_instances
+  
+  After your remote base is written, make sure to register the base outside the context
+  of the remote base, like so:
+    register_remote_base :remote_base_name
+  
+=end
 module PoolParty
 
   def register_remote_base(*args)