poolparty 0.2.18 → 0.2.69

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -4,41 +4,57 @@ module PoolParty
 
       def install_tasks
         [ 
-          setup_puppet
+          setup_poolparty,
+          setup_puppet,
+          setup_configs,
         ] << configure_tasks
       end
 
       def configure_tasks
-        [
-          setup_configs,
+        [          
           start_puppet
         ]
       end
+      
+      def setup_poolparty
+        <<-EOE
+echo "Running first time run"
+cp #{Base.remote_storage_path}/#{Base.template_directory}/puppetrunner /usr/bin/puppetrunner
+chmod +x /usr/bin/puppetrunner
+        EOE
+      end
 
       def setup_puppet
         <<-EOE
           if [ -z "$(grep -v '#' /etc/hosts | grep 'master')" ]; then echo "#{master_ip} puppet master" >> /etc/hosts; else echo "host already set"; fi
           cp #{Base.remote_storage_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
-          echo 'DAEMON_OPTS="-w 120 –fqdn #{@instance.name} –server master"' > /etc/default/puppet
         EOE
       end
 
       def setup_configs
         <<-EOS          
-          echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf        
+          echo "#{open(File.join(template_directory, "puppet.conf")).read}" > /etc/puppet/puppet.conf
+          /etc/init.d/puppetmaster stop #{unix_hide_string}
+          rm -rf /etc/puppet/ssl
+          # /usr/bin/puppetrerun
         EOS
       end
-
+      
       # /etc/init.d/puppetmasterd stop
       # puppetd --listen --fqdn #{@instance.name}
       def start_puppet
         <<-EOS
-          ps aux | grep "puppetmasterd" | awk '{print $2}' | xargs kill
-          rm -rf /etc/puppet/ssl*
-          puppetd --test  2>&1 &
+/bin/sh /usr/bin/puppetrunner
         EOS
       end
       
+      def last_install_tasks
+        [
+          "rm -rf /etc/puppet/ssl",
+          "/usr/bin/puppetrerun"
+        ]
+      end
+      
       def master_ip
         @cloud.master.ip
       end

data/lib/poolparty/modules/cloud_dsl.rb

@@ -0,0 +1,13 @@
+module PoolParty
+  module CloudDsl
+        
+    def mount_ebs_volume_at(id="", loc="/data")
+      ebs_volume_id id
+      ebs_volume_mount_point loc
+      ebs_volume_device "/dev/#{id.sanitize}"
+      has_directory(:name => loc)
+      has_mount(:name => loc, :device => ebs_volume_device)
+    end
+    
+  end
+end

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -3,10 +3,26 @@ require "ftools"
 module PoolParty
   module CloudResourcer
     
+    def plugin_directory(*args)
+      args = ["/plugins"] if args.empty?
+      args.each {|arg| Dir["#{arg}/*/*.rb"].each {|f| require f rescue "" }}
+    end
+    
     # Store block
     def store_block(&block)
-      @store_block ||= block
+      @stored_block ||= block ? block : nil
+    end
+    
+    def stored_block
+      @stored_block
+    end
+    
+    # This will run the blocks after they are stored if there is a block
+    # associated
+    def run_stored_block
+      self.run_in_context @stored_block if @stored_block
     end
+    
     # Set instances with a range
     def instances(arg)
       if arg.is_a?(Range)
@@ -65,15 +81,30 @@ module PoolParty
       ]
     end
     
-    # Set the parent on the resource
-    def set_parent(pare, sink_options=true)
-      @parent = pare      
-      # Add self as a service on the parent
-      pare.add_service(self) if pare.respond_to?(:add_service)
-      # Take the options of the parents
-      configure(pare.options) if pare.respond_to?(:options) && sink_options
+    def context_stack
+      @@context_stack ||= []
+    end
+    
+    def run_setup(parent, should_set_parent=true, &block)
+      context_stack.push parent
+      
+      set_parent if should_set_parent
+      run_in_context self, &block if block
+      
+      context_stack.pop
     end
     
+    # Set the parent on the resource
+    def set_parent(sink_options=true)
+      unless context_stack.last.nil?
+        @parent = context_stack.last
+        # Add self as a service on the parent
+        parent.add_service(self) if parent.respond_to?(:add_service)
+        # Take the options of the parents
+        configure(parent.options) if parent && parent.respond_to?(:options) && sink_options
+      end
+    end
+            
     def number_of_resources
       arr = resources.map do |n, r|
         r.size
@@ -85,5 +116,17 @@ module PoolParty
       @parent ||= nil
     end
     
+    def plugin_store
+      @plugin_store ||= []
+    end
+    
+    def realize_plugins!
+      plugin_store.each {|plugin| plugin.realize! if plugin }
+    end
+    
+    def plugin_store
+      @plugins ||= []
+    end
+    
   end
 end

data/lib/poolparty/modules/configurable.rb

@@ -21,7 +21,7 @@ module PoolParty
       end
       
       def set_vars_from_options(opts={})
-        opts.each {|k,v| self.send k.to_sym, v } unless opts.empty?
+        opts.each {|k,v| self.send k.to_sym, send_if_method(v) } unless opts.empty?
       end
     end
     

data/lib/poolparty/modules/definable_resource.rb

@@ -16,7 +16,7 @@ module PoolParty
     # 
     # For example usage, see lib/poolparty/plugins/line.rb
     def define_resource(name, &block)
-      symc = "#{name}".classify
+      symc = "#{name}".camelcase
       klass = symc.class_constant(PoolParty::Resources::CustomResource, {:preserve => true}, &block)
       PoolParty::Resources.module_eval &block
       klass
@@ -44,7 +44,7 @@ module PoolParty
     # 
     # An example is included in the poolparty-apache-plugin
     def virtual_resource(name=:virtual_resource, opts={}, &block)
-      symc = "#{name}".classify
+      symc = "#{name}".camelcase
       eval <<-EOE
         class PoolParty::Resources::#{symc} < PoolParty::Resources::Resource
         end

data/lib/poolparty/modules/file_writer.rb

@@ -5,11 +5,27 @@ module PoolParty
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )      
       FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
+    def cleanup_storage_directory
+      Dir["#{Base.storage_directory}/**/*"].each do |f|
+        ::FileUtils.rm f if ::File.file?(f)
+      end
+    end
     def copy_template_to_storage_directory(file)
       make_template_directory
       path = ::File.join( Base.tmp_path, Base.template_directory, ::File.basename(file) )
       FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
+    def copy_directory_into_template_storage_directory(dir)
+      path = make_template_directory(dir)
+      Dir["#{dir}/*"].each do |file|
+        FileUtils.cp file, path unless ::File.exists?(::File.join(path, ::File.basename(file)))
+      end
+      ::File.basename(path)
+    end
+    def make_directory_in_storage_directory(dirname="newdir")
+      path = ::File.join( Base.storage_directory, dirname )
+      make_base_path path
+    end
     def write_to_file_in_storage_directory(file, str, preceded="", &block)
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       write_to_file(path, str, &block)
@@ -34,7 +50,8 @@ module PoolParty
     end
     def make_base_path(path)
       unless FileTest.directory?(path)
-        begin          
+        begin
+          ::FileUtils.rm path if ::File.file?(path)
           ::FileUtils.mkdir_p path
         rescue Errno::ENOTDIR
         rescue Errno::EEXIST
@@ -43,14 +60,28 @@ module PoolParty
       end
     end
     def make_base_directory
-      FileUtils.mkdir_p Base.storage_directory unless ::File.directory?(Base.storage_directory)
+      begin
+        FileUtils.mkdir_p Base.storage_directory unless ::File.directory?(Base.storage_directory)
+      rescue Errno::EEXIST
+        FileUtils.rm Base.storage_directory
+        make_base_directory
+      end            
     end
-    def make_template_directory
-      path = ::File.join(Base.tmp_path, Base.template_directory)
-      FileUtils.mkdir_p path unless ::File.directory?(path)
+    def make_template_directory(dir=nil)
+      path = dir ? ::File.join(Base.tmp_path, Base.template_directory, ::File.basename(dir)) : ::File.join(Base.tmp_path, Base.template_directory)
+      begin
+        make_base_directory
+        FileUtils.mkdir path unless ::File.directory?(path)
+      rescue Errno::EEXIST
+        FileUtils.rm path if ::File.exists?(path)
+        make_template_directory(dir)
+      end      
+      path
     end
     def clear_base_directory
-      FileUtils::rm_rf "#{Base.storage_directory}"
+      Dir["#{Base.storage_directory}/**/*"].each do |f|
+        ::FileUtils.rm f if ::File.file?(f)
+      end
     end
   end
 end

data/lib/poolparty/modules/method_missing_sugar.rb

@@ -16,7 +16,7 @@ module PoolParty
     # See get_from_options for more information
     def method_missing(m, *args, &block)
       if block_given?
-        (args[0].class == self.class) ? args[0].instance_eval(&block) : super
+        (args[0].class == self.class) ? args[0].run_in_context(&block) : super
       else        
         get_from_options(m, *args, &block)
       end
@@ -42,10 +42,15 @@ module PoolParty
         if options.has_key?(m)
           options[m]
         else
-          (parent.nil? || parent.class == self.class || !parent.respond_to?(:options) || parent.options.has_key?(m)) ? nil : parent.send(m, *args, &block)
+          (parent.nil? || parent.class == self.class || !parent.respond_to?(:options) || parent.options.has_key?(m) || !parent.respond_to?(m)) ? nil : parent.send(m, *args, &block)
         end        
       else
-        options[m] = (args.is_a?(Array) && args.size > 1) ? args : args[0]
+        options[m] = 
+        if (args.is_a?(Array) && args.size > 1)
+          args
+        else
+          args[0]
+        end
       end
     end
     

data/lib/poolparty/modules/pretty_printer.rb

@@ -2,6 +2,7 @@ module PoolParty
   module PrettyPrinter
     
     def pretty_print(pre="\t")
+      pre = "\t" unless pre.is_a?(String)
       returning Array.new do |out|
         out << pretty_name(pre, self)
         out << pretty_options(pre, self)

data/lib/poolparty/modules/resourcing_dsl.rb

@@ -6,7 +6,8 @@ module PoolParty
       str ? options.merge!(:require => str) : options[:require]
     end
     def requires(str=nil)
-      str ? options.append!(:require => str) : options[:require]
+      # str ? options.append!(:require => str) : options[:require]
+      str ? options.append!(:require => send_if_method(str)) : options[:require]
     end 
     def ensures(str="running")
       if %w(absent running).map {|a| self.send a.to_sym}.include?(str)
@@ -49,7 +50,8 @@ module PoolParty
     def template(file, opts={})
       raise TemplateNotFound.new("no template given") unless file
       raise TemplateNotFound.new("template cannot be found #{file}") unless ::File.file?(file)
-      unless opts[:just_copy]          
+      
+      unless opts[:just_copy]
         options.merge!({:content => "template(\"#{::File.basename(file)}\")"})
         options.delete(:source) if options.has_key?(:source)
         copy_template_to_storage_directory(file)

data/lib/poolparty/modules/thread_pool.rb

@@ -0,0 +1,106 @@
+require 'thread'
+begin
+  require 'fasttread'
+rescue LoadError
+end
+
+class ThreadPool
+  class Worker
+    def initialize(thread_queue)
+      @mutex = Mutex.new
+      @cv = ConditionVariable.new
+      @queue = thread_queue
+      @running = true
+      @thread = Thread.new do
+        @mutex.synchronize do
+          while @running
+            @cv.wait(@mutex)
+            block = get_block
+            if block
+              @mutex.unlock
+              block.call
+              @mutex.lock
+              reset_block
+            end
+            @queue << self
+          end
+        end
+      end
+    end
+
+    def name
+      @thread.inspect
+    end
+
+    def get_block
+      @block
+    end
+
+    def set_block(block)
+      @mutex.synchronize do
+        raise RuntimeError, "Thread already busy." if @block
+        @block = block
+        # Signal the thread in this class, that there's a job to be done
+        @cv.signal
+      end
+    end
+
+    def reset_block
+      @block = nil
+    end
+
+    def busy?
+      @mutex.synchronize { !@block.nil? }
+    end
+
+    def stop
+      @mutex.synchronize do
+        @running = false
+        @cv.signal
+      end
+      @thread.join
+    end
+  end
+
+  attr_accessor :max_size
+
+  def initialize(max_size = 10)
+    @max_size = max_size
+    @queue = Queue.new
+    @workers = []
+  end
+
+  def size
+    @workers.size
+  end
+
+  def busy?
+    @queue.size < @workers.size
+  end
+
+  def shutdown
+    @workers.each { |w| w.stop }
+    @workers = []
+  end
+
+  alias :join :shutdown
+
+  def process(block=nil,&blk)
+    block = blk if block_given?
+    worker = get_worker
+    worker.set_block(block)
+  end
+
+  private
+
+  def get_worker
+    if !@queue.empty? or @workers.size == @max_size
+      return @queue.pop
+    else
+      worker = Worker.new(@queue)
+      @workers << worker
+      worker
+    end
+  end
+
+end

data/lib/poolparty/monitors/base_monitor.rb

@@ -1,14 +1,58 @@
 =begin rdoc
   Monitor class
   
-  TODO: Fill this out
+  Monitors are the basis for PoolParty scaling. Your cloud will expand and
+  contract against these monitors. You can set your cloud to be monitored by these
+  monitors simply by using them in the contract_when and the expand_when macros 
+  on your cloud like so:
+  
+    expand_when "cpu > 1.2", "memory > 0.94"
+    contract_when "cpu < 0.4", "memory < 0.3"
+  
+  You can also add your own monitors simply by creating a directory in the same
+  directory as the pool spec (the same directory as the plugin directory exists) and
+  placing your monitor file (format: [monitorname]_monitor.rb) there.
+  
+  Monitors are simply classes of the name of the monitor. They subclass the BaseMonitor
+  class from PoolParty. A sample monitor would look similar to:
+    
+    class SampleMonitor < PoolParty::Monitors::BaseMonitor
+      def run
+      end
+    end
+    register_monitor :sample
+  
+  The monitor class must have an instance level method called run. This method is called when 
+  the cloud is checking the monitor. The output of this method should be the output of
+  the monitor.
+  
+  Notice that at the end, you must call register_monitor :monitorname. This will tell your cloud
+  that it can monitor it with this monitor.
 =end
+require "#{::File.dirname(__FILE__)}/../pool/base"
+
 module PoolParty
-  module Monitors
+  module Monitors    
+    
+    module ClassMethods
+    end
+    
+    module InstanceMethods
+      def expand_when(*arr)
+        @expand_when ||= ((arr && arr.empty?) ? options[:expand_when] : configure(:expand_when => self.class.send(:rules,:expand_when,arr,false)))
+      end
+      
+      def contract_when(*arr)
+        @contract_when ||= ((arr&&arr.empty?) ? options[:contract_when] : configure(:contract_when => self.class.send(:rules,:contract_when,arr,false)))
+      end
+    end
     
     def self.register_monitor(*args)
       args.each do |arg|
-        (available_monitors << "#{arg}".downcase.to_sym)
+        (available_monitors << "#{arg}".downcase.to_sym unless available_monitors.include?("#{arg}".downcase.to_sym))
+        
+        InstanceMethods.module_eval "def #{arg}; @#{arg} ||= get_live_#{arg}; end"
+        InstanceMethods.module_eval "def get_live_#{arg}; messenger_send!(\"get_current_load #{arg}\").to_f rescue -1.0; end"
       end
     end
 
@@ -17,14 +61,29 @@ module PoolParty
     end
     
     class BaseMonitor
-      
       def self.run
         new.run
       end
-            
+    end
+    
+    def self.included(receiver)
+      receiver.extend                 PoolParty::Monitors::ClassMethods      
+      receiver.send :include,         PoolParty::Monitors::InstanceMethods
+      receiver.send :include,         Aska
     end
     
   end
 end
 
-Dir["#{File.dirname(__FILE__)}/monitors/*.rb"].each {|f| require f}
+# Require included monitors
+Dir["#{File.dirname(__FILE__)}/monitors/*.rb"].each {|f| require f}
+# Require custom monitors
+Dir["#{PoolParty::Base.custom_monitor_directories}/*.rb"].each {|f| require f}
+
+module PoolParty
+  module Cloud
+    class Cloud
+      include PoolParty::Monitors
+    end
+  end
+end

data/lib/poolparty/monitors/monitors/cpu_monitor.rb

@@ -5,7 +5,7 @@ module PoolParty
       
       def run
         str = %x[uptime]
-        str.split(/\s+/)[-2].to_f rescue 0.0
+        str.split(/\s+/)[-1].to_f rescue 0.0
       end
             
     end

data/lib/poolparty/monitors/monitors/memory_monitor.rb

@@ -4,6 +4,34 @@ module PoolParty
     class MemoryMonitor < BaseMonitor
       
       def run
+        uname = %x[uname]
+        case uname.chomp
+        when "Darwin"
+          darwin_memory_usage
+        else
+          nix_memory_usage
+        end                
+      end
+      
+      def darwin_memory_usage
+        str = %x[vm_stat]
+        keep_array = []
+        array_of_strings = str
+        begin
+          free_memory = array_of_strings[/free:(\W*)+([0-9]+)/, 2].to_f
+          active_memory = array_of_strings[/active:(\W*)([0-9]+)/, 2].to_f
+          inactive_memory = array_of_strings[/inactive:(\W*)([0-9]+)/, 2].to_f
+
+          used_memory = (active_memory + inactive_memory)
+          total_memory = (free_memory + active_memory + inactive_memory)
+
+          used_memory / total_memory
+        rescue Exception => e
+          0.0
+        end                 
+      end
+      
+      def nix_memory_usage
         str = %x[free -m | grep -i mem]
         begin
           total_memory = str.split[1].to_f
@@ -13,7 +41,6 @@ module PoolParty
         rescue Exception => e
           0.0
         end
-        
       end
             
     end

data/lib/poolparty/monitors/monitors/web_monitor.rb

@@ -0,0 +1,18 @@
+=begin rdoc
+  Web monitor.
+=end
+module PoolParty
+  module Monitors
+    
+    class WebMonitor < BaseMonitor
+      
+      def run
+        str = %x[httperf]
+        str.split(/\s+/)[-1].to_f rescue 0.0
+      end
+            
+    end
+    
+    register_monitor :cpu
+  end
+end

data/lib/poolparty/net/messenger.rb

@@ -0,0 +1,72 @@
+=begin rdoc
+  The connection to the messenger from poolparty, the client.
+  
+  This class will setup a socket connection to the master's client
+  at the messenger_client_port
+=end
+module PoolParty
+  module Messenger
+    def with_socket(testing=false, &block)
+      host = testing ? "localhost" : (master.ip)
+      vputs "Pinging #{host} with the messenger"
+      socket = TCPSocket.open(host, Base.messenger_client_port)
+      out = yield(socket)
+      socket.close
+      out
+    end
+    # TODO: Fix cookie setting
+    def self.erl_command(hostname, extra="", min_ports=7000, max_ports=7050)
+      command_line_opts = "-pa #{append_dir}/ebin -kernel inet_dist_listen_min #{min_ports} inet_dist_listen_max #{max_ports} -sname #{hostname} -setcookie poolparty"
+      
+      "erl #{command_line_opts} #{extra}"
+    end
+    
+    def self.append_dir
+      ::File.expand_path(::File.join( ::File.dirname(__FILE__), "..", "..", "erlang/messenger" ))
+    end
+    
+    def messenger_send!(msg="get_current_load cpu", testing=false)
+      with_socket(testing) do |sock|
+        sock.send(msg, 0)
+        @str = sock.recv(2000)
+      end
+      vputs "Received #{@str} from #{msg}"
+      @str
+    end
+    
+    def messenger_cast!(msg="force_reconfig")
+      with_socket do |sock|
+        sock.send(msg, 0)
+      end
+    end
+    
+    def self.messenger_send!(cmd="", testing=false)      
+      command = Messenger.erl_command("client#{Time.now.to_i}", "-s pm_client -run pm_client #{cmd} -s erlang halt -noshell")
+      testing ? command : %x[#{command}]
+    end
+    
+    # Helper methods
+    def self.startup_remote_messenger(hostname, testing=false)
+      # messenger_send!("")
+      # testing ? command : %x[#{command}]
+    end
+    
+  end
+end
+
+module PoolParty
+  module Cloud
+    class Cloud
+      include PoolParty::Messenger
+      
+      def get_current_nodes
+        nodes = messenger_send!("get_current_nodes")
+        nodes.split(" ").map {|a| a.split(/@/)[-1] }
+      end
+      
+      def reconfigure_cloud!(msg="force_reconfig")
+        messenger_cast!(msg)
+      end
+    end
+  end
+end

data/lib/poolparty/net/remote.rb

@@ -7,16 +7,19 @@ module PoolParty
     include PoolParty::Remote::Remoter
     
     def using(t)
+      @cloud = self
       if available_bases.include?(t.to_sym)
         unless using_remoter? || t.nil?
           self.class.send :attr_reader, :remote_base
+          self.class.send :attr_reader, :parent_cloud
           mod = "#{t}".preserved_module_constant
           
-          mod.send :include, PoolParty::Remote::RemoterBase
+          # mod.send :include, PoolParty::Remote::RemoterBase
           self.class.send :include, mod
           self.extend mod
-                    
+          
           @remote_base = "#{t}".preserved_module_constant
+          @parent_cloud = @cloud
         end
       else
         puts "Unknown remote base"