poolparty 0.2.18 → 0.2.69

data/lib/poolparty/net/remoter.rb

@@ -7,13 +7,12 @@ require File.dirname(__FILE__) + "/../helpers/provisioner_base"
 module PoolParty
   module Remote
     module Remoter
-      def rsync_storage_files_to_command(remote_instance)
-        if remote_instance
-          "#{rsync_command} #{Base.storage_directory}/ #{remote_instance.ip}:#{Base.remote_storage_path}"
-        end
+      def rsync_storage_files_to_command(remote_instance)        
+        "#{rsync_command} #{Base.storage_directory}/ #{remote_instance.ip}:#{Base.remote_storage_path}" if remote_instance
       end
       def run_command_on_command(cmd="ls -l", remote_instance=nil)
-        "#{ssh_command(remote_instance)} '#{cmd}'"
+        vputs "Running #{cmd} on #{remote_instance.name == %x[hostname].chomp ? "self (master)" : "#{remote_instance.name}"}"
+        remote_instance.name == %x[hostname].chomp ? %x[#{cmd}] : "#{ssh_command(remote_instance)} '#{cmd}'"
       end
       def ssh_command(remote_instance)
         "#{ssh_string} #{remote_instance.ip}"
@@ -30,31 +29,16 @@ module PoolParty
         ["-o StrictHostKeyChecking=no", "-l '#{Base.user}'", '-i "'+full_keypair_path+'"']
       end
       def rsync_command
-        "rsync --delete -azP --exclude cache -e '#{ssh_string}'"
-      end
-      # Open the cached local copy of the instances list and 
-      # create a new RemoteInstance from each line
-      def list_from_local
-        list_file = get_working_listing_file
-        if list_file
-          out = returning Array.new do |instances|
-            open(list_file).read.split("\n").each do |line|
-              instances << RemoteInstance.new(line)
-            end
-          end
-        else
-          out = list_from_remote(:cache => true)
-        end
-        return out
-      end      
-      # List the instances that are known from the remoter_base
-      # Create a RemoteInstance for each of the instances from the hash
-      # returned by the list of instances, write them to the cached file
-      # and then return the array of instances
-      def list_from_remote(options={})
-        out_array = get_remote_nodes
-        write_to_file( local_instances_list_file_locations.first, out_array.map{|a| a.to_s }.join("\n")) if options[:cache]
-        out_array
+        "rsync -azP --exclude cache -e '#{ssh_string}'"
+      end
+      def remote_ssh_array
+        ["-o StrictHostKeyChecking=no", "-l '#{Base.user}'", '-i "'+remote_keypair_path+'"']
+      end
+      def remote_ssh_string
+        (["ssh"] << remote_ssh_array).join(" ")
+      end
+      def remote_rsync_command
+        "rsync -azP --exclude cache -e '#{remote_ssh_string}'"
       end
       # Get the names of the nodes. Mainly used for puppet templating
       def list_of_node_names(options={})
@@ -64,13 +48,7 @@ module PoolParty
       def list_of_node_ips(options={})
         list_of_running_instances.collect {|ri| ri.ip }
       end
-      def get_remote_nodes
-        returning Array.new do |instances|
-          list_of_instances(respond_to?(:keypair) ? keypair : nil).each do |h|
-            instances << PoolParty::Remote::RemoteInstance.new(h)
-          end
-        end
-      end
+
       # Get the instance first instance file that exists on the system from the expected places
       # denoted in the local_instances_list_file_locations
       def get_working_listing_file
@@ -103,6 +81,14 @@ module PoolParty
         num.times {out << launch_new_instance!}
         out
       end
+      def request_launch_master_instance
+        inst = launch_new_instance!
+        wait "5.seconds"
+        when_no_pending_instances {after_launch_master(inst)}
+      end
+      def after_launch_master(h={})
+        vputs "After launch master in remoter"
+      end
       # Let's terminate an instance that is not the master instance
       def request_termination_of_non_master_instance
         inst = nonmaster_nonterminated_instances.last
@@ -110,12 +96,16 @@ module PoolParty
       end
       # Can we start a new instance?
       def can_start_a_new_instance?
-        maximum_number_of_instances_are_not_running?
+        maximum_number_of_instances_are_not_running? && list_of_pending_instances.size == 0
       end
-      # Are the maximum number of instances running?
+      # Are the maximum number of instances not running?
       def maximum_number_of_instances_are_not_running?
         list_of_running_instances.size < maximum_instances.to_i
       end
+      # Are the maximum number of instances running?
+      def maximum_number_of_instances_are_running?
+        list_of_running_instances.size >= maximum_instances.to_i
+      end
       # Launch new instance while waiting for the number of pending instances
       #  to be zero before actually launching. This ensures that we only
       #  launch one instance at a time
@@ -125,14 +115,28 @@ module PoolParty
       # A convenience method for waiting until there are no more
       # pending instances and then running the block
       def when_no_pending_instances(&block)
-        reset!
+        reset!        
         if list_of_pending_instances.size == 0
           block.call if block
         else
+          vprint "."
           wait "5.seconds"
           when_no_pending_instances(&block)
         end
       end
+      # A convenience method for waiting until all the instances have an ip
+      # assigned to them. This is useful when shifting the ip addresses
+      # around on the instances
+      def when_all_assigned_ips(&block)
+        reset!
+        if list_of_nonterminated_instances.select {|a| a.ip == "not.assigned" }.empty?          
+          block.call if block
+        else
+          vprint "."
+          wait "5.seconds"
+          when_all_assigned_ips(&block)
+        end
+      end
       
       # This will launch the minimum_instances if the minimum number of instances are not running
       # If the minimum number of instances are not running and if we can start a new instance
@@ -140,7 +144,30 @@ module PoolParty
         if can_start_a_new_instance? && !minimum_number_of_instances_are_running?         
           list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")
           reset!
-          launch_minimum_number_of_instances unless minimum_number_of_instances_are_running?
+          launch_minimum_number_of_instances
+          provision_slaves_from_n(minimum_instances.to_i)
+          after_launched
+        end
+      end
+      
+      def provision_slaves_from_n(num=1)
+        vputs "In provision_slaves_from_n: #{num}"
+        reset!
+        when_no_pending_instances do
+          vputs "Waiting for 10 seconds"
+          wait "10.seconds" # Give some time for ssh to startup          
+          @num_instances = list_of_running_instances.size
+          vputs "(@num_instances - (num))..(@num_instances): #{(@num_instances - (num))..(@num_instances)}"
+          last_instances = nonmaster_nonterminated_instances[(@num_instances - (num))..(@num_instances)]
+          last_instances.each do |inst|
+            vputs "Provision slave: #{inst}"
+            hide_output {PoolParty::Provisioner.process_clean_reconfigure_for!(inst, self)}
+            PoolParty::Provisioner.provision_slave(inst, self, false) unless inst.master? rescue vputs "Error"
+            hide_output {PoolParty::Provisioner.process_clean_reconfigure_for!(inst, self)}
+            # cmd = ". /etc/profile && cloud-provision -i #{inst.name.gsub(/node/, '')} &"
+            # vputs "Provision slave with command #{cmd}"
+          end
+          PoolParty::Provisioner.reconfigure_master(self)
         end
       end
       # Launch the master and let the master handle the starting of the cloud
@@ -149,24 +176,35 @@ module PoolParty
       # and if the master is not running AND we can start a new instance
       # Then wait for the master to launch
       def launch_and_configure_master!(testing=false)
-        request_launch_new_instances(1) if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running?
-        
-        when_no_pending_instances do
-          wait "20.seconds"
-          hide_output { Provisioner.provision_master(self, testing) }
+        vputs "Requesting to launch new instance"
+        logger.debug "Launching master"
+        request_launch_master_instance if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running? && !testing
+        reset!
+        unless testing
+          vputs ""
+          vputs "Waiting for there to be no pending instances..."
+          when_no_pending_instances do
+            when_all_assigned_ips {wait "20.seconds"}
+            vputs ""
+            vputs "Provisioning master..."
+            # cleanup_storage_directory
+            verbose ? Provisioner.provision_master(self, testing) : hide_output { Provisioner.provision_master(self, testing) }
+            verbose ? Provisioner.clear_master_ssl_certs(self, testing) : hide_output { Provisioner.clear_master_ssl_certs(self, testing) }
+          
+            after_launched
+          end
         end
-        
       end
       def is_master_running?
         !list_of_running_instances.select {|a| a.name == "master"}.first.nil?
       end
       # Stub method for the time being to handle expansion of the cloud
       def should_expand_cloud?(force=false)
-        force || false
+        valid_rules?(:expand_when) || force || false
       end
       # Stub method for the time being to handle the contraction of the cloud
       def should_contract_cloud?(force=false)
-        force || false
+        valid_rules?(:contract_when) || force || false
       end
       # Expand the cloud
       # If we can start a new instance and the load requires us to expand
@@ -176,14 +214,14 @@ module PoolParty
       # get go
       def expand_cloud_if_necessary(force=false)
         if can_start_a_new_instance? && should_expand_cloud?(force)
-          @out = request_launch_new_instances(1)
-          
-          reset!
-          when_no_pending_instances do
-            wait "20.seconds" # Give some time for ssh to startup
-            PoolParty::Provisioner.provision_slaves(self)
-            PoolParty::Provisioner.configure_master(self, testing)
-            # prepare_reconfiguration
+          vputs "Expanding the cloud based on load"
+          @num = 1
+          @num.times do |i|
+            list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")          
+            reset!
+            vputs "request_launch_new_instances: #{@num}"
+            provision_slaves_from_n(@num)
+            after_launched
           end
         end
       end
@@ -192,15 +230,29 @@ module PoolParty
       # the cloud, then we should request_termination_of_non_master_instance
       def contract_cloud_if_necessary(force=false)
         if can_shutdown_an_instance?          
-          request_termination_of_non_master_instance if should_contract_cloud?(force)
+          if should_contract_cloud?(force)
+            vputs "Shrinking the cloud by 1"
+            before_shutdown
+            request_termination_of_non_master_instance
+          end
         end
       end
       
+      # Callbacks
+      
+      # After launch callback
+      # This is called after a new instance is launched
+      def after_launched(force=false)        
+      end
+      
+      # Before shutdown callback
+      # This is called before the cloud is contracted
+      def before_shutdown
+      end
+      
       # Rsync command to the instance
       def rsync_storage_files_to(instance=nil)
-        hide_output do
-          Kernel.system "#{rsync_storage_files_to_command(instance)}" if instance
-        end
+        hide_output {Kernel.system "#{rsync_storage_files_to_command(instance)}" if instance}
       end
       # Take the rsync command and execute it on the system
       # if there is an instance given
@@ -224,12 +276,13 @@ module PoolParty
       end
       
       # Prepare reconfiguration on the master
-      # TODO: Fix the killall
       # TODO: Curious about the puppet/ssl problems...
+      # puppetd --test --no-daemonize 2>&1 &
+      # rm -rf /etc/puppet/ssl/*; 
       def prepare_reconfiguration
         unless @prepared
-          cmd = "killall ruby && rm -rf /etc/puppet/ssl/*; puppetmasterd --verbose; puppetd --test --no-daemonize 2>&1 &"
-          run_command_on(cmd, master)
+          # cmd = "/etc/init.d/puppetmaster restart"
+          # run_command_on(cmd, master)
           @prepared = true
         end
       end

data/lib/poolparty/net/remoter_base.rb

@@ -1,3 +1,21 @@
+=begin rdoc
+  The base for Remote Bases
+  
+  By extending this class, you can easily add remoters to 
+  PoolParty. There are 4 methods that the remote base needs to implement
+  in order to be compatible.
+  
+  The four methods are:
+    launch_new_instance!
+    terminate_instance(id)
+    describe_instance(id)
+    describe_instances
+  
+  After your remote base is written, make sure to register the base outside the context
+  of the remote base, like so:
+    register_remote_base :remote_base_name
+  
+=end
 module PoolParty
 
   def register_remote_base(*args)
@@ -70,27 +88,47 @@ module PoolParty
         name = (i.zero? ? "master" : "node#{i}")
         list.select {|i| i.name == name }.first
       end
-      def remote_instances_list
-        list_of_instances.map {|i| PoolParty::Remote::RemoteInstance.new(i, self) }
+      def remote_instances_list        
+        @containing_cloud = self
+        # puts "> #{@containing_cloud} #{@describe_instances.nil?}"
+        list_of_instances(keypair).collect {|h| PoolParty::Remote::RemoteInstance.new(h, @containing_cloud) }
       end
       # List the instances for the current key pair, regardless of their states
       # If no keypair is passed, select them all
       def list_of_instances(keyp=nil)
-        key = keyp ? keyp : keypair
-        describe_instances.select {|a| key ? a[:keypair] == key : true } if describe_instances
+        tmp_key = (keyp ? keyp : keypair).to_s
+        unless @describe_instances
+          tmpInstanceList = describe_instances.select {|a| a if tmp_key ? a[:keypair] == tmp_key : true }
+          has_master = !tmpInstanceList.select {|a| a[:name] == "master" }.empty?          
+          if has_master
+            @describe_instances = tmpInstanceList
+          else
+            @id = 0
+            running = select_from_instances_on_status(/running/, tmpInstanceList)
+            pending = select_from_instances_on_status(/pending/, tmpInstanceList)
+            terminated = select_from_instances_on_status(/shutting/, tmpInstanceList)
+            
+            running = running.map do |inst|
+              inst[:name] = (@id == 0 ? "master" : "node#{@id}")
+              @id += 1
+              inst
+            end.sort_by {|a| a[:index] }
+            
+            @describe_instances = [running, pending, terminated].flatten
+          end
+        end
+        @describe_instances
       end
-      # Instances
-      # Get the master from the cloud
-      def master
-        # remote_instances_list.select {|a| a.master }.first
-        @list = list_from_remote
-        @list.reject {|a| a unless a.name =~ /master/ }.first if @list.class != String
+      # Select the instances based on their status
+      def select_from_instances_on_status(status=/running/, list=[])
+        list.select {|a| a[:status] =~ status}
       end
       # Helpers
       def create_keypair
       end
       # Reset the cache of descriptions
-      def reset!
+      def reset_remoter_base!
+        @describe_instances = nil
       end
       def self.included(other)
         PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)

data/lib/poolparty/plugins/deploydirectory.rb

@@ -0,0 +1,66 @@
+=begin rdoc
+  DeployDirectory
+  
+  Deploy directory will tar.gz a local directory and sync it up to 
+  the master instance of the cloud. This enables you to send a directory
+  up to the cloud and let the master host it for the remote slaves
+=end
+module PoolParty    
+  class Deploydirectory
+        
+    virtual_resource(:deploydirectory) do
+      
+      def loaded(opts={}, parent=self)
+        package_directory        
+        unpack_directory
+        sync_directories
+      end
+            
+      def package_directory
+        path = ::File.join( Base.tmp_path, "#{::File.basename(from_dir)}.tar.gz" )
+        archive_name = "#{name.dir_safe}.tar.gz"
+        cmd = "cd #{::File.expand_path(from_dir)} && tar -czf #{archive_name} . && mv #{archive_name} #{Base.tmp_path}"
+        `#{cmd}` unless testing
+      end
+      
+      def unpack_directory
+        execute_on_master do
+          has_exec({:name => "deploy-directory-#{name}", :requires => get_directory("#{cwd}"), :cwd => cwd}) do
+            #  && rm #{Base.tmp_path}/#{parent.name.dir_safe}.tar.gz
+            command "cd #{cwd}; tar -zxf #{Base.remote_storage_path}/#{name.dir_safe}.tar.gz; rm #{Base.remote_storage_path}/#{name.dir_safe}.tar.gz"
+            onlyif "test -f #{Base.remote_storage_path}/#{name.dir_safe}.tar.gz"
+          end
+        end
+      end
+      
+      def sync_directories
+        execute_on_node do
+          has_rsyncmirror(:dir => cwd, :name => "deploydirectory-#{name}")
+        end
+      end
+      
+      def from(dir)
+        from_dir (dir.include?(" ") ? dir.gsub(/[ ]/, '') : dir)
+      end
+      
+      def to(dir)
+        cwd dir
+        has_directory(:name => "#{dir}", :requires => get_directory("#{::File.dirname(dir)}"))
+      end
+      
+      # Since git is not a native type, we have to say which core resource
+      # it is using to be able to require it
+      def class_type_name
+        "exec"
+      end
+      
+      # Because we are requiring an exec, instead of a built-in package of the git, we have to overload
+      # the to_s method and prepend it with the same name as above
+      def key
+        "deploy-directory-#{name}"
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/line.rb

@@ -4,7 +4,7 @@ module PoolParty
     define_resource(:line_in_file) do
       # Should refactor
       def has_line_in_file(line="line_in_file", file="file", opts={})
-        call_function <<-EOE
+        call_custom_function <<-EOE
         line {
           '#{file}_line' :
             file => '#{file}',
@@ -16,7 +16,7 @@ module PoolParty
       end
       
       def append_if_no_such_line(line="line", file="", refreshonly='false')
-        call_function <<-EOE
+        call_custom_function <<-EOE
         append_if_no_such_line {
           '#{file}' :
             file => '#{file}',
@@ -26,11 +26,12 @@ module PoolParty
         EOE
       end
       
-      def delete_lines_from(file="", pattern=//)
-        call_function <<-EOE
+      def delete_lines_from(file="", pattern=//, opts={})
+        call_custom_function <<-EOE
         delete_lines {
           '#{file}' :
             file => '#{file}',
+            #{opts.flush_out("\t", ",")}
             pattern => '#{pattern}'
         }
         EOE

data/lib/poolparty/plugins/rsyncmirror.rb

@@ -0,0 +1,26 @@
+module PoolParty    
+  class Rsyncmirror
+        
+    virtual_resource(:rsyncmirror) do
+      
+      def loaded(opts={}, parent=self)
+        @parent = parent
+        has_exec(opts.merge({:command => "#{cloud.remote_rsync_command} --no-implied-dirs --delete-excluded #{Base.user}@master:#{dir}/ #{dir}/".safe_quote, :name => "rsync-#{name}"}))
+      end
+            
+      # Since git is not a native type, we have to say which core resource
+      # it is using to be able to require it
+      def class_type_name
+        "exec"
+      end
+      
+      # Because we are requiring an exec, instead of a built-in package of the git, we have to overload
+      # the to_s method and prepend it with the same name as above
+      def key
+        "rsync-#{name}"
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/runit.rb

@@ -0,0 +1,96 @@
+module PoolParty
+  class Runit
+    define_resource(:runitservice) do
+      def has_runit_service(name="runitservice", downif="", templatedir="")
+        path = copy_templates_from_templatedir(templatedir)
+        call_custom_function <<-EOC
+runit_service {
+"#{name}":
+  directory => "/etc/sv",
+  downif => "/bin/ps aux | grep -v grep | grep -q #{downif}",
+  templatedir => "#{path}";  
+}        
+        EOC
+      end
+
+      def copy_templates_from_templatedir(dir=nil)
+        raise TemplateNotFound.new("template directory given") unless dir
+        raise TemplateNotFound.new("template directory cannot be found #{dir}") unless ::File.readable?(dir)
+        copy_directory_into_template_storage_directory(dir)
+      end
+
+      custom_function <<-EOF
+define runit_service ($directory = "/etc/sv", $downif = "/bin/false", $templatedir) {
+
+  file { "$directory-$name":
+    path => "$directory/$name",
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => Class["runit"]
+  }
+
+  file { "$directory/$name/log":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory-$name"]
+  }
+
+  file { "$directory/$name/log/main":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory/$name/log"]
+  }
+
+  file { "/etc/init.d/$name":
+    ensure => $lsbdistid ? {
+      'CentOS' => "/usr/local/bin/sv",
+      default => "/usr/bin/sv",
+    },
+    require => [ File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "/var/service/$name":
+    ensure => "$directory/$name",
+    require => [ File["$directory-$name"], File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "$directory/$name/log/run":
+    content => template("$templatedir/log-run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory/$name/log"],
+    notify => Service[$name]
+  }
+
+  file { "$directory/$name/run":
+    content => template("$templatedir/run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory-$name"],
+    notify => Service[$name]
+  }
+  
+  service { "$name":
+    hasrestart => true,
+    hasstatus => true,
+    require => File["/etc/init.d/$name"]
+  }
+
+  exec { "$name-down":
+    command => "/etc/init.d/$name down",
+    onlyif => $downif,
+    require => File["/etc/init.d/$name"]
+  }
+}      
+      EOF
+    end    
+  end
+end

data/lib/poolparty/plugins/svn.rb

@@ -4,7 +4,7 @@ module PoolParty
     define_resource(:svn) do
       
       def has_svnpath(opts={})
-        call_function <<-EOE
+        call_custom_function <<-EOE
         svnserve { #{opts[:name]}:
           source => "#{opts[:source]}",
           path => "#{opts[:path]}",