auser-poolparty 0.2.66 → 0.2.67

data/lib/erlang/messenger/src/pm_node.erl

@@ -18,13 +18,14 @@
          terminate/2, code_change/3]).
 
 -record(state, {
-					monitors = {}  % Tuple of monitors
+					monitors = {},  % Tuple of monitors
+					stored_loads = ?DICT:new() % Dictionary of latest loads
 				}).
 -define(SERVER, ?MODULE).
 
 % Client function definitions
 -export ([stop/0]).
--export ([get_load_for_type/1, run_cmd/1, fire_cmd/1]).
+-export ([get_current_load_for_type/1, run_cmd/1, fire_cmd/1]).
 -export ([run_reconfig/0, local_update/1, still_here/0, print_monitors/0]).
 -export ([server_location/0]).
 %%====================================================================
@@ -36,10 +37,40 @@
 
 
 % Get the load for the type sent...
-get_load_for_type(Type) ->
+get_current_load_for_type(Type) ->
 	String = string:concat(". /etc/profile && server-get-load -m ",Type),
-	{os:cmd(String)}.
-
+	LatestLoad = os:cmd(String), 
+	Load = gen_server:call(server_location(), {update_load, Type, LatestLoad}),
+	{Load}.
+
+get_load_listing(Type, State) ->
+	% Find or create the stored node
+	case ?DICT:is_key(Type, State#state.stored_loads) of
+		true -> 
+			LoadStore = ?DICT:fetch(Type, State#state.stored_loads), 
+			{LoadStore, State};
+		false -> 
+			LoadStore = [],
+			NewState = State#state{stored_loads = ?DICT:store(Type, LoadStore, State#state.stored_loads)}, 
+			{LoadStore, NewState}
+	end.
+
+get_average_load_listing(Type, State) ->
+	{LoadListing, _} = get_load_listing(Type, State),
+	utils:average_of_list(LoadListing).
+% Get the stored load for the type
+% Only store the latest NUM_LOADS_TO_STORE in the stored_loads array
+store_load_for_type(Type, Load, State) ->
+	{LoadStore, LoadListingState} = get_load_listing(Type, State),	
+	LoadList = lists:reverse(LoadStore),
+	case length(LoadList) >= ?NUM_LOADS_TO_STORE of
+		true -> [_|T] = LoadList; 
+		false -> T = LoadList
+	end,
+	NewStoredLoad = lists:append([Load], lists:reverse(T)),	
+	NewState = State#state{stored_loads = ?DICT:store(Type, NewStoredLoad, LoadListingState#state.stored_loads)},	
+	{Type, NewState}.
+	
 % Rerun the configuration
 run_reconfig() -> gen_server:cast(server_location(), {run_reconfig}).
 print_monitors() -> gen_server:call(server_location(), {print_monitors}).
@@ -56,9 +87,9 @@ stop() -> gen_server:cast(server_location(), stop).
 % Run every UPDATE_TIME seconds
 local_update(Types) ->
 	?TRACE("Updating", [?MASTER_LOCATION]),	
-	net_adm:ping(?MASTER_LOCATION),
-	Load = [{Ty, element(1, get_load_for_type(Ty))} || Ty <- Types],
-	gen_server:cast(?MASTER_SERVER, {update_node_load, node(), Load}).
+	net_adm:ping(?MASTER_LOCATION), % check in with the master
+	Load = [{Ty, element(1, get_current_load_for_type(Ty))} || Ty <- Types],
+	gen_server:cast(?MASTER_SERVER, {update_node_load, node(), Load}).	
 	
 %%--------------------------------------------------------------------
 %% Function: start_link() -> {ok,Pid} | ignore | {error,Error}
@@ -88,7 +119,8 @@ init(Args) ->
 	process_flag(trap_exit, true),
 	utils:start_timer(?UPDATE_TIME, fun() -> pm_node:local_update(Args) end),
   {ok, #state{
-					monitors=Args
+					monitors=Args,
+					stored_loads = ?DICT:new()
 				}}.
 
 %%--------------------------------------------------------------------
@@ -103,6 +135,11 @@ init(Args) ->
 handle_call({run_command, Cmd}, _From, State) ->
 	Reply = os:cmd(". /etc/profile && server-fire-cmd \""++Cmd++"\""),
 	{reply, Reply, State};
+handle_call({update_load, Type, Load}, _From, State) ->
+	{Type, NewState} = store_load_for_type(Type, Load, State),
+	Reply = get_average_load_listing(Type, NewState),
+	?TRACE("Updated load", [Reply]),
+	{reply, Reply, NewState};
 handle_call({still_there}, _From, State) ->
 	Reply = still_here,
 	{reply, Reply, State};

data/lib/erlang/messenger/src/utils.erl

@@ -35,7 +35,13 @@ tick_timer(Time, Fun) ->
 	end. 
 
 average_of_list(L) ->
-	Sum = lists:foldr( fun(Int, Sum) -> erlang:list_to_float(Int) + Sum end, 0, [F || F <- L] ),
+	Sum = lists:foldr( fun(Int, Sum) -> 
+		case erlang:is_float(Int) of
+			true -> Integer = Int;
+			false -> Integer = erlang:list_to_float(Int)
+		end,
+		Integer + Sum 
+	end, 0, [F || F <- L] ),
 	average_for_list(Sum, L).
 
 % Get the average of the list

data/lib/poolparty.rb

@@ -10,31 +10,25 @@ require 'rubygems'
   end  
 end
 
+require "#{File.dirname(__FILE__)}/poolparty/helpers/nice_printer"
+
 unless @required_software.empty?
-  @num_lines = 45
-  @centered_lines = @num_lines - 4
-  def wrap_cline(line)
-    "* #{line.center(@centered_lines)} *"
-  end
-  def wrap_lline(line)
-    "* #{line.ljust(@centered_lines)} *"
-  end
-  def header
-    "*"*@num_lines
-  end
-  empty_line = "* #{" ".ljust(@centered_lines)} *"
+  @np = NicePrinter.new(45)
+
   # error_initializing_message.txt
-  puts header
-  puts wrap_cline("Error")
-  puts wrap_lline("Missing required software")
-  puts @required_software.map {|a| wrap_lline("  #{a}") }  
-  puts wrap_lline("Please install the required software")
-  puts wrap_lline("and try again")
-  puts empty_line
-  puts wrap_lline("Try installing #{@required_software.size == 1 ? "it" : "them"} with")
-  puts @required_software.map {|a| wrap_lline("  gem install #{a}") }  
-  puts empty_line
-  puts "*"*@num_lines
+  @np.header
+  @np.center("Error")
+  @np.left("Missing required software")
+  @required_software.map {|a| @np << "  #{a}" }  
+  @np << "Please install the required software"
+  @np << "and try again"
+  @np.empty
+  @np << "Try installing #{@required_software.size == 1 ? "it" : "them"} with"
+  @required_software.map {|a| @np << "  gem install #{a}" }
+  @np.empty
+  @np.footer
+  
+  @np.print
   exit(0)
 end
 

data/lib/poolparty/base_packages/poolparty.rb

@@ -91,7 +91,7 @@ module PoolParty
           has_exec(:name => "start master messenger", :command => ". /etc/profile && server-start-master") #, :ifnot => "/bin/ps aux | /bin/grep -q pm_master"
           has_exec(:name => "start client server", :command => ". /etc/profile && server-start-client") #, :ifnot => "/bin/ps aux | /bin/grep -q client_server"
           
-          has_cron({:name => "maintain script", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3", :requires => [get_gempackage("poolparty"), get_cron("puppetd runner"), get_cron("Load handler"), get_service("haproxy")]})
+          has_cron({:name => "maintain script", :command => ". /etc/profile && cloud-maintain -n #{cloud.name}", :minute => "*/3", :requires => [get_gempackage("poolparty"), get_cron("puppetd runner"), get_cron("Load handler"), get_service("haproxy")]})
           
           has_remotefile(:name => "/usr/bin/puppetcleaner") do
             mode 744

data/lib/poolparty/core/string.rb

@@ -27,6 +27,15 @@ class String
   def sanitize
     self.gsub(/[ \.\/\-]*/, '')
   end
+  def keyerize
+    signed_short = 0x7FFFFFFF
+    len = self.sanitize.length
+    hash = 0 
+    len.times{ |i| 
+      hash = self[i] + ( hash << 6 ) + ( hash << 16 ) - hash 
+    } 
+    hash & signed_short
+  end
   def dir_safe
     self.downcase.gsub(/[ ]/, '_')
   end
@@ -65,8 +74,8 @@ class String
     klass
   end
   
-  def module_constant(&block)
-    symc = "#{self}_Module".classify
+  def module_constant(append="", &block)
+    symc = "#{self}_Module#{append}".classify
     mod = Object.const_defined?(symc) ? Object.const_get(symc.to_sym) : Module.new(&block)
     Object.const_set(symc, mod) unless Object.const_defined?(symc)
     symc.to_s.constantize

data/lib/poolparty/helpers/binary.rb

@@ -1,5 +1,22 @@
 require "ftools"
 module PoolParty
+  
+  # Load a file that contains a pool into memory
+  def load_pool(filename)
+
+    unless filename && ::File.readable?(filename)
+      puts "Please specify your cloud with -s, move it to ./pool.spec or in your POOL_SPEC environment variable"
+      exit(1)
+    else
+      $pool_specfile = filename
+      PoolParty::Script.inflate(open(filename).read, filename)
+    end
+  end
+  
+  def pool_specfile
+    $pool_specfile
+  end
+  
   module Binary
     
     class << self
@@ -26,6 +43,20 @@ module PoolParty
           f unless ::File.readable?(f)
         end.first
       end
+      # Daemonize the process
+      def daemonize(&block)
+        vputs "Daemonizing..."
+        trap("CHLD") {Process.wait(-1, Process::WNOHANG)}        
+        fork do
+          Signal.trap('HUP', 'IGNORE') # Don't die upon logout
+          File.open("/dev/null", "r+") do |devnull|
+            $stdout.reopen(devnull)
+            $stderr.reopen(devnull)
+            $stdin.reopen(devnull) unless @use_stdin
+          end
+          block.call if block
+        end
+      end
       
     end
     

data/lib/poolparty/helpers/console.rb

@@ -5,23 +5,30 @@ module PoolParty
     
     include Display
     
-    # Load a file that contains a pool into memory
-    def load_pool(filename)
-
-      unless filename && ::File.readable?(filename)
-        puts "Please specify your cloud with -s, move it to ./pool.spec or in your POOL_SPEC environment variable"
-        exit(1)
-      else
-        PoolParty::Script.inflate(open(filename).read, filename)
+    # Print help commands for the console
+    # level 0 - Basic help
+    # level 1 - Commands
+    def help(level=0)
+      @np = NicePrinter.new(60)
+      @np.header
+      @np.center "PoolParty console help"      
+      @np.center "Basics"
+      @np << "Load your pool with load_pool(filename)"
+      @np << "Reference clouds with"
+      @np << "c = cloud :cloudname"
+      @np.empty
+      if level >= 1
+        @np.center "CloudSpeak"
+        @np << "All the commands set on your cloud can be called within the console"
+        @np << "list_of_running_instances - get list of running nodes"
+        @np << "list_of_pending_instances - get list of pending nodes"
+        @np << "available_monitors - get list of the available monitors on the cloud"
+        @np << "  note: all monitors can be called as a method on the cloud"
+        @np << "    i.e. CpuMonitor makes the method cpu available on the cloud"
+        @np.empty
       end
-    end
-    
-    def extract_cloud_from_options(o)
-      o.cloudname ? [cloud(o.cloudname.downcase.to_sym)] : clouds.collect {|n,cl| cl}
-    end
-    
-    def extract_pool_from_options(o)
-      o.poolname ? [pool(o.poolname.downcase.to_sym)] : pools.collect {|n,pl| pl}
+      @np.footer
+      @np.print
     end
     
     # Clear all the pools and reload the console
@@ -29,6 +36,8 @@ module PoolParty
     # as well
     def reload!      
       reset!
+      require File.dirname(__FILE__) + "/../../../poolparty"
+      require File.dirname(__FILE__) + "/../../../poolpartycl"
     end
     
   end

data/lib/poolparty/helpers/nice_printer.rb

@@ -0,0 +1,36 @@
+class NicePrinter
+  
+  attr_accessor :num_lines
+  attr_reader :centered_lines
+  
+  def initialize(num_lines=60)
+    @num_lines = num_lines
+    @centered_lines = @num_lines - 4
+  end
+  
+  def center(line)
+    lines << "* #{line.center(@centered_lines)} *"
+  end
+  def left(line)
+    lines << "* #{line.ljust(@centered_lines)} *"
+  end
+  def header
+    lines << "*"*@num_lines
+  end
+  def footer
+    header
+  end
+  def empty
+    lines << "* #{" ".ljust(@centered_lines)} *"
+  end
+  def print
+    puts lines.join("\n")
+    ""
+  end
+  def <<(line)
+    left(line)
+  end
+  def lines
+    @lines ||= []
+  end
+end

data/lib/poolparty/helpers/optioner.rb

@@ -92,4 +92,12 @@ module PoolParty
     end
     
   end
+  
+  def extract_cloud_from_options(o)
+    o.cloudname ? [cloud(o.cloudname.downcase.to_sym)] : clouds.collect {|n,cl| cl}
+  end
+  
+  def extract_pool_from_options(o)
+    o.poolname ? [pool(o.poolname.downcase.to_sym)] : pools.collect {|n,pl| pl}
+  end
 end

data/lib/poolparty/helpers/provisioner_base.rb

@@ -52,10 +52,11 @@ module PoolParty
       
       include Configurable
       include CloudResourcer
+      include FileWriter
       
-      def initialize(instance,cloud=self, os=:ubuntu)
+      def initialize(instance,cld=self, os=:ubuntu)
         @instance = instance
-        @cloud = cloud
+        @cloud = cld
         
         options(cloud.options) if cloud && cloud.respond_to?(:options)
         set_vars_from_options(instance.options) unless instance.nil? || !instance.options || !instance.options.empty?
@@ -88,7 +89,7 @@ module PoolParty
       def process_install!(testing=false)
         error unless valid?
         write_install_file
-        setup_runner(@cloud)
+        setup_runner
         
         unless testing
           vputs "Logging on to #{@instance.ip} (#{@instance.name})"
@@ -128,7 +129,7 @@ module PoolParty
       def process_configure!(testing=false)
         error unless valid?
         write_configure_file
-        setup_runner(@cloud)
+        setup_runner
         
         unless testing
           vputs "Logging on to #{@instance.ip}"
@@ -168,7 +169,7 @@ module PoolParty
       # Tasks that need to be performed everytime we do any
       # remote ssh'ing into any instance
       def setup_runner(force=false)
-        @cloud.prepare_to_configuration
+        @cloud.prepare_for_configuration
         @cloud.build_and_store_new_config_file(force)
       end
       def valid?
@@ -283,6 +284,7 @@ module PoolParty
       def fix_rubygems
         <<-EOE
           #{installer_for("ruby rubygems")}
+          gem update --system # Force rubygems update
           echo '#{open(::File.join(template_directory, "gem")).read}' > /usr/bin/gem
         EOE
       end

data/lib/poolparty/helpers/provisioners/master.rb

@@ -192,7 +192,7 @@ cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes/poolpa
       def restart_puppetd
         <<-EOS
 echo "Running puppet manifest"
-/bin/sh /usr/bin/puppetrunner
+/usr/bin/puppetrunner
         EOS
       end
       def last_install_tasks

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -50,7 +50,8 @@ chmod +x /usr/bin/puppetrunner
       
       def last_install_tasks
         [
-          "rm -rf /etc/puppet/ssl"
+          "rm -rf /etc/puppet/ssl",
+          "/usr/bin/puppetrerun"
         ]
       end
       

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -121,7 +121,7 @@ module PoolParty
     end
     
     def realize_plugins!
-      plugin_store.each {|plugin| plugin.realize! }
+      plugin_store.each {|plugin| plugin.realize! if plugin }
     end
     
     def plugin_store

data/lib/poolparty/modules/file_writer.rb

@@ -5,6 +5,11 @@ module PoolParty
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )      
       FileUtils.cp file, path unless file == path || ::File.exists?(path)
     end
+    def cleanup_storage_directory
+      Dir["#{Base.storage_directory}/**"].each do |f|
+        ::FileUtils.rm f if ::File.file?(f)
+      end
+    end
     def copy_template_to_storage_directory(file)
       make_template_directory
       path = ::File.join( Base.tmp_path, Base.template_directory, ::File.basename(file) )
@@ -17,6 +22,10 @@ module PoolParty
       end
       ::File.basename(path)
     end
+    def make_directory_in_storage_directory(dirname="newdir")
+      path = ::File.join( Base.storage_directory, dirname )
+      make_base_path path
+    end
     def write_to_file_in_storage_directory(file, str, preceded="", &block)
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       write_to_file(path, str, &block)
@@ -70,7 +79,9 @@ module PoolParty
       path
     end
     def clear_base_directory
-      FileUtils::rm_rf "#{Base.storage_directory}"
+      Dir["#{Base.storage_directory}/**"].each do |f|
+        ::FileUtils.rm f if ::File.file?(f)
+      end
     end
   end
 end