auser-poolparty 0.2.40 → 0.2.41

data/Manifest.txt

@@ -336,6 +336,7 @@ lib/poolparty/modules/pretty_printer.rb
 lib/poolparty/modules/resourcing_dsl.rb
 lib/poolparty/modules/s3_string.rb
 lib/poolparty/modules/safe_instance.rb
+lib/poolparty/modules/thread_pool.rb
 lib/poolparty/monitors/base_monitor.rb
 lib/poolparty/monitors/monitors/cpu_monitor.rb
 lib/poolparty/monitors/monitors/memory_monitor.rb

data/lib/poolparty/base_packages/poolparty.rb

@@ -56,19 +56,21 @@ module PoolParty
         # Custom run puppet to minimize footprint
         # TODO: Update the offsetted times
         execute_on_master do          
-          has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/15") do
+          has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
+            requires get_gempackage("poolparty-latest")
             command(PoolParty::Remote::RemoteInstance.puppet_master_rerun_command)
           end
         end
         execute_on_node do
           has_cron(:name => "puppetd runner", :user => Base.user, :minute => "*/5") do
+            requires get_gempackage("poolparty-latest")
             command(PoolParty::Remote::RemoteInstance.puppet_rerun_commad)
           end
         end
         
         # These are all requirements on the master
         execute_on_master do
-          has_cron({:name => "maintain script", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3"})
+          has_cron({:name => "maintain script", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3", :requires => get_gempackage("poolparty-latest")})
           # TODO: Update this so it only runs when needed
           has_exec(:name => "start master messenger", :command => ". /etc/profile && server-start-master", :requires => [get_gempackage("poolparty-latest"), get_exec("build_messenger")], :onlyif => "ps aux | grep beam | grep master")
           

data/lib/poolparty/core/string.rb

@@ -25,7 +25,7 @@ class String
     self.split("::")[-1].downcase rescue self
   end
   def sanitize
-    self.gsub(/[\.\/]*/, '')
+    self.gsub(/[ \.\/]*/, '')
   end
   def nice_runnable(quite=true)
     self.split(/ && /).join("\n")

data/lib/poolparty/helpers/provisioner_base.rb

@@ -300,6 +300,7 @@ touch /etc/apt/sources.list
 echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
 aptitude update -y #{unix_hide_string} <<heredoc
 Y
+
 heredoc
 aptitude autoclean #{unix_hide_string} 
 fi

data/lib/poolparty/helpers/provisioners/master.rb

@@ -108,17 +108,14 @@ wget http://rubyforge.org/frs/download.php/45546/rubyforge-1.0.1.gem -O rubyforg
 wget http://rubyforge.org/frs/download.php/43954/rake-0.8.3.gem -O rake.gem 2>&1
 wget http://rubyforge.org/frs/download.php/45589/sexp_processor-3.0.0.gem -O sexp_processor.gem 2>&1
 wget http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty-latest.gem?raw=true -O poolparty-latest.gem 2>&1
-wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O grempe-amazon-ec2.gem 2>&1
+wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1
 
 #{
   %w(rake lockfile rubyforge hoe ZenTest sexp_processor flexmock logging activesupport 
-      RubyInline ParseTree ruby2ruby xml-simple poolparty-latest grempe-amazon-ec2).map do |dep|
-    "gem install --ignore-dependencies -y --no-ri --no-rdoc #{dep}.gem"
+      RubyInline ParseTree ruby2ruby xml-simple poolparty-latest amazon-ec2).map do |dep|
+    "gem install --ignore-dependencies -y --no-ri --no-rdoc #{dep}.gem #{unix_hide_string}"
   end.join("\n")
 }
-
-# gem install -y --no-ri --no-rdoc  --source http://gems.github.com grempe-amazon-ec2
-# gem install -y --no-ri --no-rdoc  --source http://gems.github.com auser-poolparty
         EOE
       end
       
@@ -127,11 +124,11 @@ wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O grempe-
       def start_puppetmaster
         <<-EOS
 . /etc/profile
-ps aux | grep puppetmaster | awk '{print $2}' | xargs kill
-/etc/init.d/puppetmaster stop
+/etc/init.d/puppetmaster stop #{unix_hide_string}
+ps aux | grep puppetmaster | awk '{print $2}' | xargs kill #{unix_hide_string} # just in case
 rm -rf /etc/puppet/ssl
-puppetmasterd --verbose
-/etc/init.d/puppetmaster start
+# Start it back up
+/etc/init.d/puppetmaster start #{unix_hide_string}
         EOS
       end
 
@@ -167,7 +164,7 @@ cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolpa
       
       def restart_puppetd
         <<-EOS
-. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master  2>&1 > /dev/null
+. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master #{unix_hide_string}
         EOS
       end
     end

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -25,7 +25,7 @@ module PoolParty
       def setup_configs
         <<-EOS          
           echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf
-          /etc/init.d/puppetmaster stop
+          /etc/init.d/puppetmaster stop #{unix_hide_string}
           # /usr/bin/puppetrerun
         EOS
       end
@@ -34,8 +34,7 @@ module PoolParty
       # puppetd --listen --fqdn #{@instance.name}
       def start_puppet
         <<-EOS
-. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master  2>&1 > /dev/null
-. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master  2>&1 > /dev/null &
+. /etc/profile && /usr/sbin/puppetd --onetime --no-daemonize --logdest syslog --server master #{unix_hide_string}
         EOS
       end
       

data/lib/poolparty/modules/thread_pool.rb

@@ -0,0 +1,107 @@
+require 'thread'
+begin
+  require 'fasttread'
+rescue LoadError
+  $stderr.puts "Using the ruby-core thread implementation"
+end
+
+class ThreadPool
+  class Worker
+    def initialize(thread_queue)
+      @mutex = Mutex.new
+      @cv = ConditionVariable.new
+      @queue = thread_queue
+      @running = true
+      @thread = Thread.new do
+        @mutex.synchronize do
+          while @running
+            @cv.wait(@mutex)
+            block = get_block
+            if block
+              @mutex.unlock
+              block.call
+              @mutex.lock
+              reset_block
+            end
+            @queue << self
+          end
+        end
+      end
+    end
+
+    def name
+      @thread.inspect
+    end
+
+    def get_block
+      @block
+    end
+
+    def set_block(block)
+      @mutex.synchronize do
+        raise RuntimeError, "Thread already busy." if @block
+        @block = block
+        # Signal the thread in this class, that there's a job to be done
+        @cv.signal
+      end
+    end
+
+    def reset_block
+      @block = nil
+    end
+
+    def busy?
+      @mutex.synchronize { !@block.nil? }
+    end
+
+    def stop
+      @mutex.synchronize do
+        @running = false
+        @cv.signal
+      end
+      @thread.join
+    end
+  end
+
+  attr_accessor :max_size
+
+  def initialize(max_size = 10)
+    @max_size = max_size
+    @queue = Queue.new
+    @workers = []
+  end
+
+  def size
+    @workers.size
+  end
+
+  def busy?
+    @queue.size < @workers.size
+  end
+
+  def shutdown
+    @workers.each { |w| w.stop }
+    @workers = []
+  end
+
+  alias :join :shutdown
+
+  def process(block=nil,&blk)
+    block = blk if block_given?
+    worker = get_worker
+    worker.set_block(block)
+  end
+
+  private
+
+  def get_worker
+    if !@queue.empty? or @workers.size == @max_size
+      return @queue.pop
+    else
+      worker = Worker.new(@queue)
+      @workers << worker
+      worker
+    end
+  end
+
+end

data/lib/poolparty/net/remoter.rb

@@ -194,11 +194,9 @@ module PoolParty
             reset!
             wait "20.seconds" # Give some time for ssh to startup
             @num_instances = nonmaster_nonterminated_instances.size
-            last_instances = nonmaster_nonterminated_instances[(@num_instances - @num)...(@num_instances)]
+            last_instances = nonmaster_nonterminated_instances[(@num_instances - (@num + 1))..(@num_instances)]
             last_instances.each do |inst|
-              vputs "Provisioning #{inst.name}"
-              cmd = ". /etc/profile && cloud-provision -i #{inst.name.gsub(/node/, '')} #{unix_hide_string} &"
-              Kernel.system cmd
+              PoolParty::Provisioner.provision_slave(inst, self, false)
             end
             PoolParty::Provisioner.reconfigure_master(self, force)
             after_launched

data/lib/poolparty/version.rb

@@ -2,7 +2,7 @@ module PoolParty
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 40
+    TINY  = 41
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/poolparty.gemspec

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: poolparty
 version: !ruby/object:Gem::Version 
-  version: 0.2.40
+  version: 0.2.41
 platform: ruby
 authors: 
 - Ari Lerner
@@ -439,6 +439,7 @@ files:
 - lib/poolparty/modules/resourcing_dsl.rb
 - lib/poolparty/modules/s3_string.rb
 - lib/poolparty/modules/safe_instance.rb
+- lib/poolparty/modules/thread_pool.rb
 - lib/poolparty/monitors/base_monitor.rb
 - lib/poolparty/monitors/monitors/cpu_monitor.rb
 - lib/poolparty/monitors/monitors/memory_monitor.rb

data/spec/poolparty/net/remote_spec.rb

@@ -233,7 +233,8 @@ describe "Remote" do
         end
         it "should call a new slave provisioner" do
           @tc.stub!(:should_expand_cloud?).once.and_return true
-          Kernel.should_receive(:system).with(". /etc/profile && cloud-provision -i 5 2>&1 > /dev/null &").and_return true
+          PoolParty::Provisioner.should_receive(:provision_slave).at_least(1)
+          # Kernel.should_receive(:system).with(". /etc/profile && cloud-provision -i 5 2>&1 > /dev/null &").and_return true
         end
         it "should call reconfigure on the master to pick up the new slave" do
           @tc.stub!(:should_expand_cloud?).once.and_return true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: auser-poolparty
 version: !ruby/object:Gem::Version 
-  version: 0.2.40
+  version: 0.2.41
 platform: ruby
 authors: 
 - Ari Lerner
@@ -439,6 +439,7 @@ files:
 - lib/poolparty/modules/resourcing_dsl.rb
 - lib/poolparty/modules/s3_string.rb
 - lib/poolparty/modules/safe_instance.rb
+- lib/poolparty/modules/thread_pool.rb
 - lib/poolparty/monitors/base_monitor.rb
 - lib/poolparty/monitors/monitors/cpu_monitor.rb
 - lib/poolparty/monitors/monitors/memory_monitor.rb