auser-poolparty 0.2.60 → 0.2.61

data/Manifest.txt

@@ -35,6 +35,7 @@ bin/server-get-load
 bin/server-list-active
 bin/server-list-responding
 bin/server-rerun
+bin/server-send-command
 bin/server-show-stats
 bin/server-start-client
 bin/server-start-master
@@ -42,6 +43,7 @@ bin/server-start-node
 bin/server-stop-client
 bin/server-stop-master
 bin/server-stop-node
+bin/server-update-hosts
 config/hoe.rb
 config/requirements.rb
 examples/basic.rb

data/README.txt

@@ -41,6 +41,8 @@ sudo gem install auser-poolparty
 == TODO:
 * Replace services with Runit
 * Refactor provisioning to use erlang
+* Move the building of the resource tree on read to on resolve
+* Add queuing of tasks on the messenger
 
 == LICENSE:
 

data/bin/cloud-ensure-provisioning

@@ -23,9 +23,15 @@ o.loaded_clouds.each do |cloud|
         s << "/usr/sbin/puppetca --clean #{node}.compute-1.internal"
         s << "/usr/sbin/puppetca --clean #{node}.ec2.internal"
       end.join(";")
-      cmd = ". /etc/profile && #{str};cloud-provision -i #{node.gsub(/node/, '')};#{str}"
+  
+      node_id = node.gsub(/node/, '')
+      
+      cmd = ". /etc/profile && #{str};cloud-provision -i #{node_id};#{str}"
+      running_cmd = "ps aux | grep -v grep | grep \"cloud-provision -i #{node_id}\""
+      
       vputs "Executing #{cmd}"
-      `#{cmd}`
+      running = %x[#{running_cmd}]
+      `#{cmd}` if running.chomp.empty?
     end
     # @tp.join
   end  

data/bin/cloud-provision

@@ -4,6 +4,7 @@ require "poolparty"
 require "poolpartycl"
 
 o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
   opts.on('-p', '--slave', 'Provision slave (default: false)')    { optioner.provision_slave true }
   opts.on('-i num', '--id num', 'Instance num to provision')    { |i| optioner.instance_number i }
 end

data/bin/pool-list

@@ -15,8 +15,16 @@ o.loaded_pools.each do |pool|
     puts header("Listing pool #{name}")
     clouds.each do |name, cloud|
       puts subheader("Cloud: #{name}")
-      out = location == "remote" ? cloud.list_from_remote({:cache => true}).map{|a| a.to_s} : cloud.list_from_local.map{|a| a.to_s}
-      puts out
+      with_cloud(cloud) do
+        puts header("Listing cloud #{name}")
+        puts subheader("Active instances")
+        puts list_of_running_instances.map{|a| a.to_s}.join("\n")
+        puts ""
+        if list_of_pending_instances.size > 0
+          puts subheader("Pending instances")
+          puts list_of_pending_instances.map{|a| a.to_s}.join("\n")
+        end
+      end
     end
   end
 end

data/bin/server-send-command

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolpartycl"
+
+@command = ARGV[-1] || ""
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-c command', '--command command', 'Command to run')    { |c| optioner.command c }
+end
+
+@command,@args = Arr.split(" ")[0], Arr.split(" ")[1..-1]
+
+o.loaded_clouds do |cloud|
+  with_cloud(cloud, {:command => @command, :command_args => @args}) do
+    cloud.send command.to_sym, command_args
+  end  
+end

data/bin/server-update-hosts

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolpartycl"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n name', '--name name', 'Update hosts for local pool named [name]')    { |p| optioner.poolname p }
+end
+
+o.loaded_pools.each do |pool|
+  with_pool(pool) do
+    pool_clouds.each do |cl|
+      
+      new_hosts_listing = returning Array.new do |arr|
+        cl.list_of_running_instances.each do |ri|
+          vputs ri.hosts_file_listing_for(cl)
+          arr << ri.hosts_file_listing_for(cl)
+        end
+        cl.other_clouds.each do |other|
+          other.list_of_running_instances.each do |ri|
+            vputs ri.hosts_file_listing_for(cl)
+            arr << ri.hosts_file_listing_for(cl)
+          end
+        end
+      end.join("\n")
+      
+      if testing
+        puts new_hosts_listing
+      else
+        new_hosts_file = "#{Base.manifest_path}/classes/hosts.pp"
+        
+        arr = returning Array.new do |arr|
+          new_hosts_listing.split(/\n/).each do |line|
+            parts = line.split(/\t/).map {|a| a.chomp }
+            ip,hostname,aliases = parts[0], parts[1], parts[2..-1]
+            
+            arr << PoolParty::Resources::Host.new(:ip => ip, :name => Resolv::DNS.new.getaddress(hostname), :alias => aliases).to_string
+          end
+        end
+        
+        ::File.open(new_hosts_file, "w+") {|f| f << arr.join("\n") }
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 
 # Load required gems
 @required_software = Array.new
-%w(activesupport ftools logging ruby2ruby).each do |lib|
+%w(activesupport ftools logging resolv ruby2ruby).each do |lib|
   begin
     require lib
   rescue Exception => e

data/lib/poolparty/aska/aska.rb

@@ -9,7 +9,7 @@ module Aska
         arr.each do |line|
           next unless line
           rule = Rule.new(line)
-          raise LoadRulesException.new(line) unless rule.valid?
+          next unless rule.valid?
           k = rule.key
           v = rule.var
           m = rule.comparison

data/lib/poolparty/base_packages/haproxy.rb

@@ -2,14 +2,14 @@ module PoolParty
   class Base
     plugin :haproxy do
       
-      def enable
+      def enable        
         execute_on_master do
           has_package({:name => "haproxy"})
 
           # Restart sysklogd after we update the haproxy.log
           has_service(:name => "sysklogd")
           
-          # Template variables
+          # Template variables          
           has_variable(:name => "name_haproxy", :value => "#{cloud.name}")
           has_variable(:name => "nodenames_haproxy", :value => "generate('/usr/bin/env', '/usr/bin/server-list-active', '-c', 'name')")
           has_variable(:name => "node_ips_haproxy",  :value => "generate('/usr/bin/env', '/usr/bin/server-list-active', '-c', 'ip')")

data/lib/poolparty/base_packages/poolparty.rb

@@ -1,15 +1,9 @@
 module PoolParty
   class Base
-    plugin :poolparty do
+    plugin :poolparty_base_packages do
       
       def enable                
         # Build hostsfile
-        # TODO: COME BACK AND CLEAN THIS UP
-        (self.respond_to?(:list_of_running_instances) ? self : parent).list_of_running_instances.each do |ri|
-          has_host({:name => "#{ri.name}", :ip => ri.ip})
-        end
-
-        # has_host({:name => "$hostname", :ip => "127.0.0.1", :alias => "localhost"})
         
         has_package(:name => "erlang")
         has_package(:name => "erlang-dev")
@@ -43,16 +37,22 @@ module PoolParty
           has_gempackage(:name => "poolparty", :download_url => "http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=true", :requires => [get_gempackage("ruby2ruby"), get_gempackage("RubyInline"), get_gempackage("ParseTree")])
           
           # , :ifnot => "/bin/ps aux | /bin/grep -q pm_node"
-          has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger -v")
+          has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger")
           has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node")
+          has_exec(:name => "update_hosts", :command => ". /etc/profile && server-update-hosts")
           # has_runit_service("pm_node", "pm_node", File.join(File.dirname(__FILE__), "..", "templates/messenger/node/"))
         end
         
         # execute_on_node do
         has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
           requires get_gempackage("poolparty")
-          command "/usr/bin/puppetrun"
+          command "/usr/bin/puppetrunner"
+        end
+        has_remotefile(:name => "/usr/bin/puppetrunner") do
+          mode 744
+          template File.join(File.dirname(__FILE__), "..", "templates/puppetrunner")
         end
+        
         # end
         
         # Cloud panel setup
@@ -92,17 +92,12 @@ module PoolParty
             mode 744
             template File.join(File.dirname(__FILE__), "..", "templates/puppetcleaner")
           end
+          
+          has_remotefile(:name => "/usr/bin/puppetrerun") do
+            mode 744
+            template File.join(File.dirname(__FILE__), "..", "templates/puppetrerun")
+          end
         end        
-        
-        has_remotefile(:name => "/usr/bin/puppetrerun") do
-          mode 744
-          template File.join(File.dirname(__FILE__), "..", "templates/puppetrerun")
-        end
-        has_remotefile(:name => "/usr/bin/puppetrunner") do
-          mode 744
-          template File.join(File.dirname(__FILE__), "..", "templates/puppetrunner")
-        end
-        
         # has_host(:name => "puppet", :ip => (self.respond_to?(:master) ? self : parent).master.ip)
       end
       

data/lib/poolparty/core/object.rb

@@ -55,7 +55,7 @@ class Object
     end
   end
   def block_instance_eval(*args, &block)
-    return instance_eval(*args,&block) unless block_given? && !block.arity.zero?
+    return instance_eval(*args,&block) unless block && !block.arity.zero?
     old_method = (self.class.instance_method(:__) rescue nil)
     self.class.send(:define_method, :__, &block)
     block_method = self.class.instance_method(:__)
@@ -72,16 +72,19 @@ class Object
   def meta_undef name
     meta_eval { remove_method name }
   end
-  def run_in_context(&block)
+  def run_in_context(context=self, &block)
     name="temp_#{self.class}_#{respond_to?(:parent) ? parent.to_s : Time.now.to_i}".to_sym
     meta_def name, &block
-    self.send name, self
+    self.send name, context
     # self.instance_eval &block if block
-    meta_undef name
+    meta_undef name rescue ""
   end
   def vputs(m="", o=self)
     puts m if o.verbose
   end
+  def dputs(m="", o=self)
+    vputs(">#{::File.basename(__FILE__)} (#{__LINE__}) #{m}", o)
+  end
   def vprint(m="", o=self)
     print m if o.verbose
   end

data/lib/poolparty/helpers/console.rb

@@ -20,6 +20,10 @@ module PoolParty
       o.cloudname ? [cloud(o.cloudname.downcase.to_sym)] : [clouds[clouds.keys.first]]
     end
     
+    def extract_pool_from_options(o)
+      o.poolname ? [pool(o.poolname.downcase.to_sym)] : [pools[pools.keys.first]]
+    end
+    
     # Clear all the pools and reload the console
     # Call within console to reset and reload the entire poolparty base
     # as well

data/lib/poolparty/helpers/optioner.rb

@@ -71,6 +71,9 @@ module PoolParty
 
         reject_junk_options!
         raise CloudNotFoundException.new("Please specify your cloud with -s, move it to ./pool.spec or in your POOL_SPEC environment variable") unless loaded_clouds && !loaded_clouds.empty?
+        loaded_pools.each do |pl|
+          pl.configure(self.options)
+        end
         loaded_clouds.each do |cl|
           cl.configure(self.options)
         end

data/lib/poolparty/helpers/provisioners/master.rb

@@ -25,9 +25,9 @@ module PoolParty
           setup_autosigning,
           install_poolparty,
           setup_poolparty,
-          create_local_node,        
+          restart_puppetmaster,
           run_first_time,
-          restart_puppetmaster
+          create_local_node,
         ] << configure_tasks
       end
 
@@ -45,6 +45,7 @@ module PoolParty
         <<-EOS
 echo "Creating local host entry"
 if [ -z \"$(grep -v '#' /etc/hosts | grep 'puppet')" ]; then echo '#{@master_ip}           puppet master localhost' >> /etc/hosts; fi
+hostname master
         EOS
       end
 
@@ -116,11 +117,11 @@ wget http://rubyforge.org/frs/download.php/45546/rubyforge-1.0.1.gem -O rubyforg
 wget http://rubyforge.org/frs/download.php/43954/rake-0.8.3.gem -O rake.gem 2>&1
 wget http://rubyforge.org/frs/download.php/45589/sexp_processor-3.0.0.gem -O sexp_processor.gem 2>&1
 wget http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=true -O poolparty.gem 2>&1
-wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1
+# wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1
 
 #{
   %w(rake lockfile rubyforge hoe ZenTest sexp_processor flexmock logging activesupport 
-      RubyInline ParseTree ruby2ruby xml-simple poolparty amazon-ec2).map do |dep|
+      RubyInline ParseTree ruby2ruby xml-simple poolparty).map do |dep|
     "gem install --ignore-dependencies -y --no-ri --no-rdoc #{dep}.gem #{unix_hide_string}"
   end.join("\n")
 }
@@ -132,21 +133,24 @@ wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-
       def restart_puppetmaster
         <<-EOS
 echo "(Re)starting poolparty"
-. /etc/profile
-/etc/init.d/puppetmaster stop #{unix_hide_string}
-ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string} # just in case
-rm -rf /etc/puppet/ssl
-# Start it back up
-# puppetmasterd --verbose
+# . /etc/profile
+# /etc/init.d/puppetmaster stop #{unix_hide_string}
+# ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string} # just in case
+# rm -rf /etc/puppet/ssl
+# # Start it back up
+# # puppetmasterd --verbose
 /etc/init.d/puppetmaster start
         EOS
       end
       
       def run_first_time
 <<-EOE
-echo "#{open(File.join(template_directory, "puppetrerun")).read}" > /usr/bin/puppetrerun
+echo "Running first time run"
+cp #{Base.remote_storage_path}/#{Base.template_directory}/puppetrerun /usr/bin/puppetrerun
+cp #{Base.remote_storage_path}/#{Base.template_directory}/puppetrunner /usr/bin/puppetrunner
 chmod +x /usr/bin/puppetrerun
-/bin/sh /usr/bin/puppetrerun
+chmod +x /usr/bin/puppetrunner
+/bin/sh /usr/bin/server-update-hosts
 EOE
       end
 
@@ -163,7 +167,7 @@ node default {
 node "#{ri.name}" inherits default {}
            EOS
          end
-"echo '#{str}' > /etc/puppet/manifests/nodes/nodes.pp"
+"echo '#{str}' > #{Base.manifest_path}/../nodes/nodes.pp"
       end
 
       def move_templates
@@ -191,10 +195,9 @@ cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolpa
       end
       
       def restart_puppetd
-        terminate_string = "ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string}; puppetmasterd --verbose"
         <<-EOS
 echo "Running puppet manifest"
-/bin/sh /usr/bin/puppetrerun
+/bin/sh /usr/bin/puppetrunner
         EOS
       end
     end

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -4,6 +4,7 @@ module PoolParty
 
       def install_tasks
         [ 
+          setup_poolparty,
           setup_puppet,
           setup_configs,
           run_once_and_clean
@@ -18,7 +19,7 @@ module PoolParty
       
       def setup_poolparty
         <<-EOE
-echo "#{open(File.join(template_directory, "puppetrun")).read}" > /usr/bin/puppetrun
+echo "#{open(File.join(template_directory, "puppetrunner")).read}" > /usr/bin/puppetrunner
         EOE
       end
 

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -10,8 +10,15 @@ module PoolParty
     
     # Store block
     def store_block(&block)
-      @store_block ||= block
+      @stored_block ||= block
     end
+    
+    # This will run the blocks after they are stored if there is a block
+    # associated
+    def run_stored_block      
+      self.run_in_context @stored_block if @stored_block
+    end
+    
     # Set instances with a range
     def instances(arg)
       if arg.is_a?(Range)
@@ -70,17 +77,29 @@ module PoolParty
       ]
     end
     
+    def context_stack
+      @@context_stack ||= []
+    end
+    def run_setup(parent, should_set_parent=true, &block)
+      context_stack.push parent
+      
+      set_parent if should_set_parent
+      run_in_context self, &block if block
+      
+      context_stack.pop
+    end
+    
     # Set the parent on the resource
-    def set_parent(pare, sink_options=true)
-      unless pare == self
-        @parent = pare
+    def set_parent(sink_options=true)
+      # unless context_stack.last == self
+        @parent = context_stack.last
         # Add self as a service on the parent
-        pare.add_service(self) if pare.respond_to?(:add_service)
+        parent.add_service(self) if parent.respond_to?(:add_service)
         # Take the options of the parents
-        configure(pare.options) if pare.respond_to?(:options) && sink_options
-      end
+        configure(parent.options) if parent && parent.respond_to?(:options) && sink_options
+      # end
     end
-    
+            
     def number_of_resources
       arr = resources.map do |n, r|
         r.size