poolparty 0.2.6 → 0.2.18

data/lib/poolparty/helpers/messenger.rb

@@ -0,0 +1,29 @@
+module PoolParty
+  module Messenger
+    
+    # TODO: Fix cookie setting
+    def self.erl_command(hostname, extra="")
+      command_line_opts = "-pa #{append_dir}/ebin -sname #{hostname} -setcookie poolparty"
+      
+      "erl #{command_line_opts} #{extra}"
+    end
+    
+    def self.append_dir
+      ::File.join( ::File.dirname(__FILE__), "..", "..", "erlang/messenger" )
+    end
+    
+    def messenger_send!(cmd="", testing=false)
+      cmd = Messenger.erl_command("client", "-rsh ssh -noshell -run pm_client #{cmd} -s erlang halt")
+      testing ? cmd : Kernel.system(cmd)
+    end
+    
+  end
+end
+
+module PoolParty
+  module Cloud
+    class Cloud
+      include PoolParty::Messenger
+    end
+  end
+end

data/lib/poolparty/helpers/optioner.rb

@@ -9,7 +9,7 @@ module PoolParty
     include MethodMissingSugar
         
     def initialize(args=[], opts={}, &block)
-      @arguments = args
+      @arguments = parse_args(args)
       @parse_options = opts[:parse_options] ? opts[:parse_options] : true
       
       set_default_options
@@ -17,6 +17,10 @@ module PoolParty
       self
     end
     
+    def parse_args(argv, safe=[])
+      argv
+    end
+    
     def parent
       self
     end
@@ -47,7 +51,7 @@ module PoolParty
         exit
       end
       
-      opts.parse!(@arguments)
+      opts.parse(@arguments.dup)
       
       process_options      
       output_options if verbose

data/lib/poolparty/helpers/provisioner_base.rb

@@ -17,7 +17,7 @@ module PoolParty
     end
 
     def self.provision_slaves(cloud, testing=false)
-      cloud.nonmaster_nonterminated_instances.each do |sl|
+      cloud.nonmaster_nonterminated_instances.each do |sl|        
         provision_slave(sl, cloud, testing)
       end
     end
@@ -53,7 +53,7 @@ module PoolParty
         loaded
       end
       # Callback after initialized
-      def loaded      
+      def loaded(opts={}, parent=self)      
       end
       # This is the actual runner for the installation    
       def install
@@ -71,10 +71,11 @@ module PoolParty
         setup_runner(@cloud)
         
         unless testing
-          puts "Logging on to #{@instance.ip}"
+          puts "Logging on to #{@instance.ip}" if verbose
+          @cloud.prepare_reconfiguration
           @cloud.rsync_storage_files_to(@instance)
           
-          cmd = "cd #{Base.remote_storage_path}/#{Base.tmp_path} && chmod +x install_#{name}.sh && /bin/sh install_#{name}.sh && rm -rf *"
+          cmd = "cd #{Base.remote_storage_path} && chmod +x install_#{name}.sh && /bin/sh install_#{name}.sh && rm install_#{name}.sh"
           hide_output do
             @cloud.run_command_on(cmd, @instance)
           end          
@@ -94,9 +95,10 @@ module PoolParty
         setup_runner(@cloud)        
         
         unless testing
+          puts "Logging on to #{@instance.ip}" if verbose
           @cloud.rsync_storage_files_to(@instance)
 
-          cmd = "cd #{Base.remote_storage_path}/#{Base.tmp_path} && chmod +x configure_#{name}.sh && /bin/sh configure_#{name}.sh && rm -rf *"
+          cmd = "cd #{Base.remote_storage_path} && chmod +x configure_#{name}.sh && /bin/sh configure_#{name}.sh && rm configure_#{name}.sh"
           @cloud.run_command_on(cmd, @instance)
         end
       end
@@ -136,7 +138,8 @@ module PoolParty
       def default_install_tasks
         [
           upgrade_system,
-          install_puppet_master,
+          fix_rubygems,
+          install_puppet,
           custom_install_tasks
         ] << install_tasks
       end
@@ -179,7 +182,7 @@ module PoolParty
       # Package installers for general *nix operating systems
       def self.installers
         @installers ||= {
-          :ubuntu => "apt-get install -y",
+          :ubuntu => "aptitude install -y",
           :fedora => "yum install",
           :gentoo => "emerge"
         }
@@ -203,6 +206,10 @@ module PoolParty
       def template_directory
         File.join(File.dirname(__FILE__), "..", "templates")
       end
+      
+      def fix_rubygems
+        "echo '#{open(::File.join(template_directory, "gem")).read}' > /usr/bin/gem"
+      end
 
       def create_local_node
         str = <<-EOS
@@ -224,21 +231,21 @@ module PoolParty
           "          
           touch /etc/apt/sources.list
           echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
-          apt-get update --fix-missing -y
-          apt-get upgrade -y
+          aptitude update -y
+          aptitude autoclean
           "
         else
           "# No system upgrade needed"
         end
       end
       
-      def install_puppet_master
+      def install_puppet
         "#{installer_for( puppet_packages )}"
       end
 
       def create_poolparty_manifest
         <<-EOS
-          mv #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes
+          cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes
         EOS
       end
     end

data/lib/poolparty/helpers/provisioners/master.rb

@@ -29,7 +29,7 @@ module PoolParty
 
       def configure_tasks
         [
-          start_puppetmaster,
+          # start_puppetmaster,
           create_local_node,
           move_templates,
           create_poolparty_manifest,
@@ -37,14 +37,7 @@ module PoolParty
         ]
       end
       
-      # def install_haproxy
-      #   "#{installer_for( "haproxy" )}"
-      # end
-      # 
-      # def install_heartbeat        
-      #   "#{installer_for( "heartbeat-2" )}"
-      # end
-
+      # If the master is not in the hosts file, then add it to the hosts file
       def create_local_hosts_entry
         <<-EOS
 if [ -z \"$(grep -v '#' /etc/hosts | grep 'master')" ]; then echo '#{@master_ip}           puppet master' >> /etc/hosts; fi
@@ -59,7 +52,7 @@ echo "import 'nodes/*.pp'" > /etc/puppet/manifests/site.pp
 echo "import 'classes/*.pp'" >> /etc/puppet/manifests/site.pp
 mkdir -p /etc/puppet/manifests/nodes 
 mkdir -p /etc/puppet/manifests/classes
-mv #{Base.remote_storage_path}/#{Base.tmp_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
+cp #{Base.remote_storage_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
         EOS
       end
 
@@ -73,7 +66,7 @@ echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/
         <<-EOS
 echo "
 [files]
-  path #{Base.remote_storage_path}/#{Base.tmp_path}
+  path #{Base.remote_storage_path}
   allow *" > /etc/puppet/fileserver.conf
 mkdir -p /var/poolparty/facts
 mkdir -p /var/poolparty/files
@@ -104,31 +97,40 @@ node "#{ri.name}" inherits default {}
       def move_templates
         <<-EOS
 mkdir -p #{Base.template_path}
-mv #{Base.remote_storage_path}/#{Base.template_directory}/* #{Base.template_path}
+cp #{Base.remote_storage_path}/#{Base.template_directory}/* #{Base.template_path}
         EOS
       end
-
+      
       def create_poolparty_manifest
         <<-EOS
-mv #{Base.remote_storage_path}/#{Base.tmp_path}/poolparty.pp /etc/puppet/manifests/classes/poolparty.pp
-mv #{Base.remote_storage_path}/#{Base.tmp_path}/#{Base.key_file_locations.first} "#{Base.base_config_directory}/.ppkeys"
-mv #{Base.remote_storage_path}/#{Base.tmp_path}/#{Base.default_specfile_name} #{Base.base_config_directory}/#{Base.default_specfile_name}
-mv #{Base.remote_storage_path}/#{Base.tmp_path}/#{@cloud.full_keypair_name} #{@cloud.remote_keypair_path}
+mv #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolparty.pp
+cp #{Base.remote_storage_path}/#{Base.key_file_locations.first} "#{Base.base_config_directory}/.ppkeys"
+cp #{Base.remote_storage_path}/#{Base.default_specfile_name} #{Base.base_config_directory}/#{Base.default_specfile_name}
+#{copy_ssh_app}
         EOS
       end
+      
+      def copy_ssh_app
+        if @cloud.remote_keypair_path != "#{Base.remote_storage_path}/#{@cloud.full_keypair_name}"
+          "cp #{Base.remote_storage_path}/#{@cloud.full_keypair_name} #{@cloud.remote_keypair_path}"
+        end
+      end
 
+      # ps aux | grep puppetmasterd | awk '{print $2}' | xargs kill
+      # rm -rf /etc/puppet/ssl
+      # puppetmasterd --verbose
       def start_puppetmaster
         <<-EOS
-ps aux | grep puppetmasterd | awk '{print $2}' | xargs kill
-rm -rf /etc/puppet/ssl
-puppetmasterd
         EOS
       end
 
+      # puppetd --listen --fqdn #{@instance.name}
       def restart_puppetd
         <<-EOS
-          puppetd --listen --fqdn=#{@instance.name} --server=master
-          puppetrun --host #{@instance.name}
+          killall ruby
+          rm -rf /etc/puppet/ssl/*
+          puppetmasterd --verbose
+          . /etc/profile && #{@instance.puppet_runner_command}
         EOS
       end
     end

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -17,21 +17,25 @@ module PoolParty
 
       def setup_puppet
         <<-EOE
-          puppetd --mkusers
           if [ -z "$(grep -v '#' /etc/hosts | grep 'master')" ]; then echo "#{master_ip} puppet master" >> /etc/hosts; else echo "host already set"; fi
-          mv #{Base.remote_storage_path}/#{Base.tmp_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
+          cp #{Base.remote_storage_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
+          echo 'DAEMON_OPTS="-w 120 –fqdn #{@instance.name} –server master"' > /etc/default/puppet
         EOE
       end
 
       def setup_configs
-        <<-EOS
+        <<-EOS          
           echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf        
         EOS
       end
 
+      # /etc/init.d/puppetmasterd stop
+      # puppetd --listen --fqdn #{@instance.name}
       def start_puppet
         <<-EOS
-puppetd --listen --fqdn=#{@instance.name}
+          ps aux | grep "puppetmasterd" | awk '{print $2}' | xargs kill
+          rm -rf /etc/puppet/ssl*
+          puppetd --test  2>&1 &
         EOS
       end
       

data/lib/poolparty/modules/definable_resource.rb

@@ -52,6 +52,7 @@ module PoolParty
       klass = "PoolParty::Resources::#{symc}".constantize
       klass.module_eval &block if block
       klass.send :define_method, :virtual_resource?, Proc.new{true}
+      klass.send :define_method, :printable?, Proc.new{false}
       klass
     end
     

data/lib/poolparty/modules/file_writer.rb

@@ -1,21 +1,21 @@
 module PoolParty
   module FileWriter
-    def copy_file_to_storage_directory(file)
+    def copy_file_to_storage_directory(file, preceded="")
       make_base_directory
-      path = ::File.join( Base.storage_directory, ::File.basename(file) )
-      FileUtils.cp file, path
+      path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )      
+      FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
     def copy_template_to_storage_directory(file)
       make_template_directory
-      path = ::File.join( Base.template_directory, ::File.basename(file) )
-      FileUtils.cp file, path
+      path = ::File.join( Base.tmp_path, Base.template_directory, ::File.basename(file) )
+      FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
-    def write_to_file_in_storage_directory(file, str, &block)
-      path = ::File.join( Base.storage_directory, ::File.basename(file) )
+    def write_to_file_in_storage_directory(file, str, preceded="", &block)
+      path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       write_to_file(path, str, &block)
     end
-    def write_to_file(file, str, &block)
-      path = ::File.join( Base.storage_directory, ::File.basename(file) )
+    def write_to_file(file, str, preceded="", &block)
+      path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       make_base_path( Base.storage_directory )
       ::File.open(path, "w+") do |f|
         f.print str
@@ -46,7 +46,8 @@ module PoolParty
       FileUtils.mkdir_p Base.storage_directory unless ::File.directory?(Base.storage_directory)
     end
     def make_template_directory
-      FileUtils.mkdir_p Base.template_directory unless ::File.directory?(Base.template_directory)      
+      path = ::File.join(Base.tmp_path, Base.template_directory)
+      FileUtils.mkdir_p path unless ::File.directory?(path)
     end
     def clear_base_directory
       FileUtils::rm_rf "#{Base.storage_directory}"

data/lib/poolparty/modules/method_missing_sugar.rb

@@ -42,7 +42,7 @@ module PoolParty
         if options.has_key?(m)
           options[m]
         else
-          (parent.nil? || parent.class == self.class || !parent.respond_to?(:options)) ? nil : parent.send(m, *args, &block)
+          (parent.nil? || parent.class == self.class || !parent.respond_to?(:options) || parent.options.has_key?(m)) ? nil : parent.send(m, *args, &block)
         end        
       else
         options[m] = (args.is_a?(Array) && args.size > 1) ? args : args[0]

data/lib/poolparty/modules/pretty_printer.rb

@@ -1,20 +1,20 @@
 module PoolParty
   module PrettyPrinter
     
-    def pretty_print(prev="\t")
+    def pretty_print(pre="\t")
       returning Array.new do |out|
-        out << pretty_name(prev, self)
-        out << pretty_options(prev, self)
+        out << pretty_name(pre, self)
+        out << pretty_options(pre, self)
         
         if self.respond_to?(:clouds)
           clouds.each do |name, cl|
-            out << pretty_name(prev*2, cl)
-            out << pretty_options(prev*2, cl) #cl.pretty_print("#{prev}\t")
+            out << pretty_name(pre*2, cl)
+            out << pretty_options(pre*2, cl) #cl.pretty_print("#{pre}\t")
           end
         end
         if self.respond_to?(:plugins)
-          out << "#{prev}\t\tPlugins"
-          out << "#{prev}\t\t" + plugins.map {|a| a}.join("\n")
+          out << "#{pre}\t\tPlugins"
+          out << "#{pre}\t\t" + plugins.map {|a| a}.join("\n")
         end
       end.join("\n")
     end
@@ -22,17 +22,17 @@ module PoolParty
     # Gather options on the object
     # Do not print if the option is nil or empty.
     # Also, don't show the option if the option is empty or the default option on the cloud    
-    def pretty_options(prev, o)
+    def pretty_options(pre, o)
       return "" unless o.respond_to?(:options)
       print_options = (o.respond_to?(:parent) && o.parent && o.parent.respond_to?(:options)) ? 
         (o.options.delete_if {|k,v| o.parent.options.has_key?(k) && o.parent.options[k] == o.options[k] && !o.options[k].nil? } ) : 
         o.options
       print_options = print_options.map {|k,v| [k, o.send(k.to_sym).to_s] }.inject({}) { |r,e| r[e[0]] = e[1] unless o.class.default_options[e[0]] == e[1] || e[1].nil? || e[1].empty?; r }
-      print_options.flush_out("#{prev}\t")
+      print_options.flush_out("#{pre}\t")
     end
     
-    def pretty_name(prev, o)
-      "#{prev}#{o.class.to_s.top_level_class.capitalize}: #{o.name if o.respond_to?(:name)}"
+    def pretty_name(pre, o)
+      "#{pre}#{o.class.to_s.top_level_class.capitalize}: #{o.name if o.respond_to?(:name)}"
     end
     
   end

data/lib/poolparty/modules/resourcing_dsl.rb

@@ -0,0 +1,61 @@
+module PoolParty
+  module ResourcingDsl
+    # Overrides for syntax
+    # Allows us to send require to require a resource
+    def require(str="")
+      str ? options.merge!(:require => str) : options[:require]
+    end
+    def requires(str=nil)
+      str ? options.append!(:require => str) : options[:require]
+    end 
+    def ensures(str="running")
+      if %w(absent running).map {|a| self.send a.to_sym}.include?(str)
+        str == "absent" ? is_absent : is_present
+      else
+        options.append!(:ensure => str)
+      end
+      str
+    end
+    # Allows us to send an ensure to ensure the presence of a resource
+    def is_present(*args)
+      options.merge!(:ensure => present)
+    end
+    # Ensures that what we are sending is absent
+    def is_absent(*args)
+      options.merge!(:ensure => absent)
+    end
+    # Alias for unless
+    def ifnot(str="")
+      options.merge!(:unless => str)
+    end
+    def present
+      "present"
+    end
+    def absent
+      "absent"
+    end
+    def cancel(*args)
+      options[:cancelled] = args.empty? ? true : args[0]
+    end
+    def cancelled?
+      options[:cancelled] || false
+    end
+    # Give us a template to work with on the resource
+    # Make sure this template is moved to the tmp directory as well
+    # 
+    # TODO: Change this method to store the template files for later
+    # copying to prevent unnecessary copying and tons of directories
+    # everywhere
+    def template(file, opts={})
+      raise TemplateNotFound.new("no template given") unless file
+      raise TemplateNotFound.new("template cannot be found #{file}") unless ::File.file?(file)
+      unless opts[:just_copy]          
+        options.merge!({:content => "template(\"#{::File.basename(file)}\")"})
+        options.delete(:source) if options.has_key?(:source)
+        copy_template_to_storage_directory(file)
+      else
+        copy_file_to_storage_directory(file)
+      end
+    end
+  end
+end

data/lib/poolparty/monitors/base_monitor.rb

@@ -6,11 +6,25 @@
 module PoolParty
   module Monitors
     
+    def self.register_monitor(*args)
+      args.each do |arg|
+        (available_monitors << "#{arg}".downcase.to_sym)
+      end
+    end
+
+    def self.available_monitors
+      $available_monitors ||= []
+    end
+    
     class BaseMonitor
       
-      
-      
+      def self.run
+        new.run
+      end
+            
     end
     
   end
-end
+end
+
+Dir["#{File.dirname(__FILE__)}/monitors/*.rb"].each {|f| require f}

data/lib/poolparty/monitors/monitors/cpu_monitor.rb

@@ -0,0 +1,15 @@
+module PoolParty
+  module Monitors
+    
+    class CpuMonitor < BaseMonitor
+      
+      def run
+        str = %x[uptime]
+        str.split(/\s+/)[-2].to_f rescue 0.0
+      end
+            
+    end
+    
+    register_monitor :cpu
+  end
+end

data/lib/poolparty/monitors/monitors/memory_monitor.rb

@@ -0,0 +1,23 @@
+module PoolParty
+  module Monitors
+    
+    class MemoryMonitor < BaseMonitor
+      
+      def run
+        str = %x[free -m | grep -i mem]
+        begin
+          total_memory = str.split[1].to_f
+          used_memory = str.split[2].to_f
+
+          used_memory / total_memory
+        rescue Exception => e
+          0.0
+        end
+        
+      end
+            
+    end
+    
+    register_monitor :memory
+  end
+end

data/lib/poolparty/net/remote_instance.rb

@@ -61,7 +61,12 @@ module PoolParty
       # Printing. This is how we extract the instances into the listing on the 
       # local side into the local listing file
       def to_s
-        "#{name} #{ip}"
+        "#{name}\t#{ip}"
+      end
+      
+      # Commands for the servers
+      def puppet_runner_command
+        ". /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master 2>&1"
       end
     end
     

data/lib/poolparty/net/remoter.rb

@@ -9,7 +9,7 @@ module PoolParty
     module Remoter
       def rsync_storage_files_to_command(remote_instance)
         if remote_instance
-          "#{rsync_command} #{Base.storage_directory} #{remote_instance.ip}:#{Base.remote_storage_path}"
+          "#{rsync_command} #{Base.storage_directory}/ #{remote_instance.ip}:#{Base.remote_storage_path}"
         end
       end
       def run_command_on_command(cmd="ls -l", remote_instance=nil)
@@ -152,7 +152,7 @@ module PoolParty
         request_launch_new_instances(1) if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running?
         
         when_no_pending_instances do
-          wait "10.seconds"
+          wait "20.seconds"
           hide_output { Provisioner.provision_master(self, testing) }
         end
         
@@ -180,8 +180,10 @@ module PoolParty
           
           reset!
           when_no_pending_instances do
-            @ri = list_of_running_instances.last
-            PoolParty::Provisioner.provision_slave(@ri, self, !force)
+            wait "20.seconds" # Give some time for ssh to startup
+            PoolParty::Provisioner.provision_slaves(self)
+            PoolParty::Provisioner.configure_master(self, testing)
+            # prepare_reconfiguration
           end
         end
       end
@@ -195,7 +197,7 @@ module PoolParty
       end
       
       # Rsync command to the instance
-      def rsync_storage_files_to(instance=nil)        
+      def rsync_storage_files_to(instance=nil)
         hide_output do
           Kernel.system "#{rsync_storage_files_to_command(instance)}" if instance
         end
@@ -215,6 +217,22 @@ module PoolParty
       def ssh_into_instance_number(num=0)
         ssh_into( get_instance_by_number( num || 0 ) )
       end
+      
+      # Run command on the instance by the number
+      def run_command_on_instance_number(cmd="ls -l", num=0)
+        run_command_on(cmd, get_instance_by_number( num || 0 ) )
+      end
+      
+      # Prepare reconfiguration on the master
+      # TODO: Fix the killall
+      # TODO: Curious about the puppet/ssl problems...
+      def prepare_reconfiguration
+        unless @prepared
+          cmd = "killall ruby && rm -rf /etc/puppet/ssl/*; puppetmasterd --verbose; puppetd --test --no-daemonize 2>&1 &"
+          run_command_on(cmd, master)
+          @prepared = true
+        end
+      end
 
       def self.included(receiver)
         receiver.extend self

data/lib/poolparty/net/remoter_base.rb

@@ -8,7 +8,7 @@ module PoolParty
   
   def remote_bases
     $remote_bases ||= []
-  end  
+  end
 
   module Remote    
     # This class is the base class for all remote types
@@ -61,6 +61,10 @@ module PoolParty
       def list_of_nonterminated_instances(list = remote_instances_list)
         list.reject {|i| i.terminating? || i.terminated? }
       end
+      # get the master instance
+      def master
+        get_instance_by_number(0)
+      end
       # Get instance by number
       def get_instance_by_number(i=0, list = remote_instances_list)
         name = (i.zero? ? "master" : "node#{i}")