auser-poolparty 0.2.16 → 0.2.20

data/lib/erlang/messenger/src/pm_cluster.erl

@@ -1,22 +1,33 @@
--module(cluster).
+-module(pm_cluster).
 
 -ifdef(EUNIT).
 -include_lib("eunit/include/eunit.hrl").
 -endif.
 
--export([slaves/1]).
-	%% Argument:
-	%% Hosts: List of hostname (string)
+-export ([slaves/1, slaves/0]).
+-export ([get_live_nodes/0,refresh_live_nodes/0]).
+-export ([send_call/2, master/0]).
+
+send_call(Type, Args) ->
+	Nodes = get_live_nodes(),
+	rpc:multicall(Nodes, pm_node, Type, [Args]).
+
+master() ->
+	{erlang:node()}.
+
 slaves([]) ->
 	ok;
 
 slaves([Host|Hosts]) ->
 	Args = erl_system_args(),
-	NodeName = "cluster",
+	NodeName = "pp_cluster",
 	{ok, Node} = slave:start_link(Host, NodeName, Args),
 	io:format("Erlang node started = [~p]~n", [Node]),
 	slaves(Hosts).
 
+slaves() ->
+	get_live_nodes().
+
 erl_system_args()->
 	Shared = case init:get_argument(shared) of
 		error -> " ";
@@ -27,8 +38,20 @@ erl_system_args()->
 		atom_to_list(erlang:get_cookie()),
 		Shared, " +Mea r10b "
 	]).
-%% Do not forget to start erlang with a command like:
-%% erl -rsh ssh -sname clustmaster
 
-basic_test_() ->
-	?_assert(true).
+refresh_live_nodes() ->
+	String = ". /etc/profile && server-list-active -c name",
+	Nodes = string:tokens(os:cmd(String), "\n\t"),
+	io:format("nodes: ~p~n", [Nodes]),
+	lists:map(
+		fun(No) ->
+			io:format("pinging ~p~n", [list_to_atom(lists:append("pp_cluster@",No))]),
+			net_adm:ping(list_to_atom(lists:append("@pp_cluster", No)))
+		end,
+		Nodes),
+	ok.
+% Get the live nodes
+get_live_nodes() ->
+	nodes().
+%% Do not forget to start erlang with a command like:
+%% erl -rsh ssh -sname clustmaster

data/lib/erlang/messenger/src/pm_master.erl

@@ -18,7 +18,8 @@
 -define(SERVER, ?MODULE).
 
 % Client function definitions
--export ([get_load/1, reconfigure_cloud/0, fire_cmd/1, get_live_nodes/0]).
+-export ([get_load/1, reconfigure_cloud/0, fire_cmd/1]).
+-export ([shutdown_cloud/0]).
 
 %%====================================================================
 %% API
@@ -28,7 +29,7 @@
 %% Description: Starts the server
 %%--------------------------------------------------------------------
 start_link() ->
-  gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+  gen_server:start_link({global, ?SERVER}, ?MODULE, [], []).
 
 %%====================================================================
 %% gen_server callbacks
@@ -98,22 +99,20 @@ code_change(_OldVsn, State, _Extra) ->
 
 % pm_master:get_load("0", "cpu").
 get_load(Type) ->
-	Nodes = get_live_nodes(),
-	{Loads, _} = rpc:multicall(Nodes, pm_node, get_load_for_type, [Type]),
+	{Loads, _} = pm_cluster:send_call(get_load_for_type, [Type]),
 	{utils:convert_responses_to_int_list(Loads)}.
 
 % Send reconfigure tasks to every node
 reconfigure_cloud() ->
-	Nodes = get_live_nodes(),
-	{_, _} = rpc:multicall(Nodes, pm_node, run_reconfig, []),
+	pm_cluster:send_call(run_reconfig, []),
 	{ok}.
 
 % Fire the given command on all nodes
 fire_cmd(Cmd) ->
-	Nodes = get_live_nodes(),
-	{_, _} = rpc:multicall(Nodes, pm_node, fire_cmd, [Cmd]),
+	pm_cluster:send_call(fire_cmd, [Cmd]),
 	{ok}.
 
-% Get the live nodes
-get_live_nodes() ->
-	nodes().
+% Shutdown
+shutdown_cloud() ->
+	pm_cluster:send_call(stop, []),
+	{ok}.

data/lib/erlang/messenger/src/pm_node.erl

@@ -34,8 +34,8 @@
 %% Fires a ping every 10 seconds
 %%--------------------------------------------------------------------
 start_link() ->
-	utils:start_timer(10000, fun() -> net_adm:ping(master@auser) end),
-  gen_server:start_link({local, ?SERVER}, ?MODULE, [], []).
+	utils:start_timer(10000, fun() -> net_adm:ping(master) end),
+  gen_server:start_link({global, ?SERVER}, ?MODULE, [], []).
 
 %%====================================================================
 %% gen_server callbacks
@@ -117,7 +117,6 @@ run_reconfig() ->	{os:cmd(". /etc/profile && server-rerun")}.
 % Allows us to fire off any command (allowed by poolparty on the check)
 fire_cmd(Cmd) ->
 	String = ". /etc/profile && server-fire-cmd \""++Cmd++"\"",
-	io:format("Sending command "++String),
 	{os:cmd(String)}.
 
 % Stop the pm_node entirely

data/lib/erlang/messenger/src/pm_packager.erl

@@ -0,0 +1,73 @@
+-module (pm_packager).
+-compile(export_all).
+-behaviour(application).
+
+%% Application callbacks
+-export([start/2, stop/1]).
+
+start() ->
+	start("", "").
+%%====================================================================
+%% Application callbacks
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: start(Type, StartArgs) -> {ok, Pid} |
+%%                                     {ok, Pid, State} |
+%%                                     {error, Reason}
+%% Description: This function is called whenever an application
+%% is started using application:start/1,2, and should start the processes
+%% of the application. If the application is structured according to the
+%% OTP design principles as a supervision tree, this means starting the
+%% top supervisor of the tree.
+%%--------------------------------------------------------------------
+start(_, _) ->
+	recompile_scripts("0.1").
+
+%%--------------------------------------------------------------------
+%% Function: stop(State) -> void()
+%% Description: This function is called whenever an application
+%% has stopped. It is intended to be the opposite of Module:start/2 and
+%% should do any necessary cleaning up. The return value is ignored.
+%%--------------------------------------------------------------------
+stop(_State) ->
+  ok.
+
+%%====================================================================
+%% Internal functions
+%%====================================================================
+
+gen_rel(Name, Vers) ->
+		RelName = lists:append(["pm_",Name,"_rel-",Vers]),
+    F = lists:append(["{release, {\"",Name,"\",\"",Vers,"\"}, ",
+                      "{erts,\"",erlang:system_info(version),"\"},"
+                      "[{kernel,\"",get_vsn(kernel),"\"},",
+                      "{stdlib,\"",get_vsn(stdlib),"\"},",
+                      "{inets,\"",get_vsn(inets),"\"},",
+                      "{crypto,\"",get_vsn(crypto),"\"},",
+                      "{sasl,\"",get_vsn(sasl),"\"},",
+                      "{",Name,",\"",Vers,"\"}]}."]),
+	file:write_file( lists:append(["ebin/", RelName, ".rel"]),F).
+
+% Recompiles the boot scripts
+recompile_scripts(Vers) ->
+	gen_rel("master", Vers),
+	gen_rel("node", Vers),
+	systools:make_script("pm_node_rel-"++Vers, [local,{path,["ebin"]}]),
+	systools:make_script("pm_master_rel-"++Vers, [local,{path,["ebin"]}]).
+
+package_scripts(Vers) ->
+	systools:make_tar("ebin/pm_node_rel-"++Vers),
+	systools:make_tar("ebin/pm_master_rel-"++Vers).
+
+get_vsn(Module) ->
+    AppFile = code:lib_dir(Module)++"/ebin/"++atom_to_list(Module)++".app",
+    {ok,[{application,_App,Attrs}]} = file:consult(AppFile),
+    {value,{vsn,Vsn}} = lists:keysearch(vsn,1,Attrs),
+    Vsn.
+
+install_messenger(Vers) ->
+	Root = code:root_dir(),
+	io:format("~p root: "++Root, [Vers]),
+	ok.
+
+% Tests

data/lib/erlang/messenger/src/utils.erl

@@ -13,9 +13,9 @@ convert_responses_to_int_list(L) ->
 
 % Start a timer to fire off Fun after Time number of milliseconds
 start_timer(Time, Fun) -> 
-	register(clock, spawn(fun() -> tick_timer(Time, Fun) end)). 
+	register(?MODULE, spawn(fun() -> tick_timer(Time, Fun) end)). 
 
-stop_timer() -> clock ! stop. 
+stop_timer() -> ?MODULE ! stop. 
 
 tick_timer(Time, Fun) -> 
 	receive 
@@ -35,11 +35,4 @@ average_for_list(Num, L) ->
 			0;
 		_ ->
 			Num / length(L)
-	end.
-
-% Recompiles the boot scripts
-recompile_scripts(Vers) ->
-	systools:make_script("pm_node_rel-"++Vers, [local]),
-	systools:make_script("pm_master_rel-"++Vers, [local]).
-
-% Tests
+	end.

data/lib/poolparty/base_packages/heartbeat.rb

@@ -6,28 +6,28 @@ module PoolParty
         execute_if("$hostname", "master") do
           has_package(:name => "heartbeat-2", :ensure => "installed") do
             # These can also be passed in via hash
+            has_service(:name => "heartbeat", :hasstatus => true)
+            
             has_remotefile(:name => "/etc/ha.d/ha.cf") do
               mode 444
-              notify 'Service["heartbeat"]'
+              notify service(:name => "heartbeat")
               template File.join(File.dirname(__FILE__), "..", "templates/ha.cf")
             end
+            
+            has_exec(:name => "heartbeat-update-cib", :command => "/usr/sbin/cibadmin -R -x /etc/ha.d/cib.xml", :refreshonly => true)
 
             has_remotefile(:name => "/etc/ha.d/authkeys") do
               mode 400
-              notify 'Service["heartbeat"]'
+              notify service(:name => "heartbeat")
               template File.join(File.dirname(__FILE__), "..", "templates/authkeys")
             end
 
             has_remotefile(:name => "/etc/ha.d/cib.xml") do
               mode 444
-              notify 'Exec["heartbeat-update-cib"]'
+              notify exec(:name => "heartbeat-update-cib")
               template File.join(File.dirname(__FILE__), "..", "templates/cib.xml")
-            end
-            
-            has_service(:name => "heartbeat", :hasstatus => true)
-          end          
-        
-          has_exec(:name => "heartbeat-update-cib", :command => "/usr/sbin/cibadmin -R -x /etc/ha.d/cib.xml", :refreshonly => true)
+            end            
+          end
           
           if @parent.provisioning?
             variable(:name => "ha_nodenames", :value => "#{list_of_running_instances.map{|a| "#{a.send :name}" }.join("\t")}")

data/lib/poolparty/base_packages/poolparty.rb

@@ -4,25 +4,29 @@ module PoolParty
       
       def enable        
         has_package(:name => "erlang")
+        has_package(:name => "erlang-dev")
+        has_package(:name => "erlang-src")
         
         has_package(:name => "rubygems") do
           # These should be installed automagically by poolparty, but just in case
           # TODO: Fix the requires method with a helper          
           has_gempackage(:name => "logging")
-          has_gempackage(:name => "xml-simple") do
+          has_gempackage(:name => "xml-simple", :source => "/var/poolparty/xml-simple.gem") do
             has_gempackage(:name => "grempe-amazon-ec2", :source => "http://gems.github.com")
           end
           
-          has_gempackage(:name => "hoe") do
-            has_gempackage(:name => "open4")
-            
-            has_gempackage(:name => "ParseTree", :version => "2.2.0") do
-              has_gempackage(:name => "ruby2ruby")
-              has_gempackage(:name => "activesupport") do
-                has_gempackage(:name => "auser-poolparty", :source => "http://gems.github.com")
+          has_gempackage(:name => "open4")
+          
+          has_gempackage(:name => "ParseTree", :version => "2.2.0") do
+            has_gempackage(:name => "ruby2ruby")
+            has_gempackage(:name => "activesupport") do
+              has_gempackage(:name => "auser-poolparty", :source => "http://gems.github.com") do
+                has_exec(:name => "build_messenger", :command => ". /etc/profile && server-build-messenger") do
+                  has_exec(:name => "start_node", :command => ". /etc/profile && server-start-node")
+                end                
               end
-              has_gempackage(:name => "RubyInline")
             end
+            has_gempackage(:name => "RubyInline")
           end          
           
         end
@@ -41,7 +45,9 @@ module PoolParty
         
         # These are all requirements on the master
         execute_if("$hostname", "master") do
-          has_cron({:command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3"})
+          has_cron({:name => "maintain script ", :command => ". /etc/profile && which cloud-maintain | /bin/sh", :minute => "*/3"})
+          # TODO: Update this so it only runs when needed
+          has_exec(:name => ". /etc/profile && server-start-master")
         end
         # has_host(:name => "puppet", :ip => (self.respond_to?(:master) ? self : parent).master.ip)
       end

data/lib/poolparty/config/allowed_commands.yml

@@ -0,0 +1 @@
+echo 'hello world' > test.del.txt

data/lib/poolparty/core/object.rb

@@ -26,6 +26,14 @@ class Object
     block.in_context(self).call
     self
   end
+  def send_if_method(v, *args)
+    if (v.nil? || v.to_s.empty? || v.is_a?(Array) || v.is_a?(Integer))
+      v
+    else
+      vs = v.to_s.to_sym
+      respond_to?(vs) ? self.send(vs, *args) : v
+    end
+  end
   def to_option_string
     case self.class
     when String

data/lib/poolparty/exceptions/UnacceptableCommand.rb

@@ -0,0 +1,5 @@
+class UnacceptableCommand < Exception  
+  def initialize(msg="Unacceptable command")
+    @message = "Disallowed or unacceptable command error: #{msg}"
+  end
+end

data/lib/poolparty/helpers/messenger.rb

@@ -0,0 +1,29 @@
+module PoolParty
+  module Messenger
+    
+    # TODO: Fix cookie setting
+    def self.erl_command(hostname, extra="")
+      command_line_opts = "-pa #{append_dir}/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname #{hostname} -setcookie poolparty"
+      
+      "erl #{command_line_opts} #{extra} 2>&1 &"
+    end
+    
+    def self.append_dir
+      ::File.join( ::File.dirname(__FILE__), "..", "..", "erlang/messenger" )
+    end
+    
+    def messenger_send!(cmd="", testing=false)
+      cmd = Messenger.erl_command("client", "-rsh ssh -noshell -run pm_client #{cmd} -s erlang halt")
+      testing ? cmd : Kernel.system(cmd)
+    end
+    
+  end
+end
+
+module PoolParty
+  module Cloud
+    class Cloud
+      include PoolParty::Messenger
+    end
+  end
+end

data/lib/poolparty/helpers/provisioner_base.rb

@@ -71,9 +71,9 @@ module PoolParty
         setup_runner(@cloud)
         
         unless testing
-          puts "Logging on to #{@instance.ip}" if verbose
-          @cloud.prepare_reconfiguration
+          puts "Logging on to #{@instance.ip}" if verbose          
           @cloud.rsync_storage_files_to(@instance)
+          @cloud.prepare_reconfiguration
           
           cmd = "cd #{Base.remote_storage_path} && chmod +x install_#{name}.sh && /bin/sh install_#{name}.sh && rm install_#{name}.sh"
           hide_output do
@@ -92,7 +92,7 @@ module PoolParty
       def process_configure!(testing=false)
         error unless valid?
         write_configure_file
-        setup_runner(@cloud)        
+        setup_runner(@cloud)
         
         unless testing
           puts "Logging on to #{@instance.ip}" if verbose
@@ -229,10 +229,10 @@ module PoolParty
         case @os
         when :ubuntu
           "          
-          touch /etc/apt/sources.list
-          echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
-          aptitude update -y
-          aptitude autoclean
+touch /etc/apt/sources.list
+echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
+# aptitude update -y 2>&1
+# aptitude autoclean 2>&1 &
           "
         else
           "# No system upgrade needed"

data/lib/poolparty/helpers/provisioners/master.rb

@@ -17,8 +17,6 @@ module PoolParty
 
       def install_tasks
         [
-          install_haproxy,
-          install_heartbeat,
           create_local_hosts_entry,
           setup_basic_structure,
           setup_configs,        
@@ -40,18 +38,16 @@ module PoolParty
       # If the master is not in the hosts file, then add it to the hosts file
       def create_local_hosts_entry
         <<-EOS
-if [ -z \"$(grep -v '#' /etc/hosts | grep 'master')" ]; then echo '#{@master_ip}           puppet master' >> /etc/hosts; fi
+if [ -z \"$(grep -v '#' /etc/hosts | grep 'puppet')" ]; then echo '#{@master_ip}           puppet master' >> /etc/hosts; fi
         EOS
       end
 
       def setup_basic_structure
         <<-EOS
-puppetmasterd --mkusers
-mkdir -p #{Base.remote_storage_path}
-echo "import 'nodes/*.pp'" > /etc/puppet/manifests/site.pp
-echo "import 'classes/*.pp'" >> /etc/puppet/manifests/site.pp
 mkdir -p /etc/puppet/manifests/nodes 
 mkdir -p /etc/puppet/manifests/classes
+echo "import 'nodes/*.pp'" > /etc/puppet/manifests/site.pp
+echo "import 'classes/*.pp'" >> /etc/puppet/manifests/site.pp
 cp #{Base.remote_storage_path}/namespaceauth.conf /etc/puppet/namespaceauth.conf
         EOS
       end

data/lib/poolparty/helpers/provisioners/slave.rb

@@ -36,6 +36,7 @@ module PoolParty
           ps aux | grep "puppetmasterd" | awk '{print $2}' | xargs kill
           rm -rf /etc/puppet/ssl*
           puppetd --test  2>&1 &
+          rm -rf /etc/puppet/ssl*
         EOS
       end
       

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -67,11 +67,13 @@ module PoolParty
     
     # Set the parent on the resource
     def set_parent(pare, sink_options=true)
-      @parent = pare      
-      # Add self as a service on the parent
-      pare.add_service(self) if pare.respond_to?(:add_service)
-      # Take the options of the parents
-      configure(pare.options) if pare.respond_to?(:options) && sink_options
+      unless pare == self
+        @parent = pare
+        # Add self as a service on the parent
+        pare.add_service(self) if pare.respond_to?(:add_service)
+        # Take the options of the parents
+        configure(pare.options) if pare.respond_to?(:options) && sink_options
+      end
     end
     
     def number_of_resources

data/lib/poolparty/modules/configurable.rb

@@ -21,7 +21,7 @@ module PoolParty
       end
       
       def set_vars_from_options(opts={})
-        opts.each {|k,v| self.send k.to_sym, v } unless opts.empty?
+        opts.each {|k,v| self.send k.to_sym, send_if_method(v) } unless opts.empty?
       end
     end
     

data/lib/poolparty/modules/method_missing_sugar.rb

@@ -45,7 +45,12 @@ module PoolParty
           (parent.nil? || parent.class == self.class || !parent.respond_to?(:options) || parent.options.has_key?(m)) ? nil : parent.send(m, *args, &block)
         end        
       else
-        options[m] = (args.is_a?(Array) && args.size > 1) ? args : args[0]
+        options[m] = 
+        if (args.is_a?(Array) && args.size > 1)
+          args
+        else
+          args[0]
+        end
       end
     end
     

data/lib/poolparty/modules/pretty_printer.rb

@@ -2,6 +2,7 @@ module PoolParty
   module PrettyPrinter
     
     def pretty_print(pre="\t")
+      pre = "\t" unless pre.is_a?(String)
       returning Array.new do |out|
         out << pretty_name(pre, self)
         out << pretty_options(pre, self)

data/lib/poolparty/modules/resourcing_dsl.rb

@@ -6,7 +6,8 @@ module PoolParty
       str ? options.merge!(:require => str) : options[:require]
     end
     def requires(str=nil)
-      str ? options.append!(:require => str) : options[:require]
+      # str ? options.append!(:require => str) : options[:require]
+      str ? options.append!(:require => send_if_method(str)) : options[:require]
     end 
     def ensures(str="running")
       if %w(absent running).map {|a| self.send a.to_sym}.include?(str)

data/lib/poolparty/monitors/monitors/cpu_monitor.rb

@@ -5,7 +5,7 @@ module PoolParty
       
       def run
         str = %x[uptime]
-        str.split(/\s+/)[-3].to_f rescue 0.0
+        str.split(/\s+/)[-2].to_f rescue 0.0
       end
             
     end

data/lib/poolparty/net/remoter.rb

@@ -225,9 +225,11 @@ module PoolParty
       
       # Prepare reconfiguration on the master
       # TODO: Fix the killall
+      # TODO: Curious about the puppet/ssl problems...
+      # puppetd --test --no-daemonize 2>&1 &
       def prepare_reconfiguration
         unless @prepared
-          cmd = "killall ruby && rm -rf /etc/puppet/ssl/*; puppetmasterd --verbose; puppetd --test --no-daemonize 2>&1 &"
+          cmd = "killall ruby 2>&1 && rm -rf /etc/puppet/ssl/*; puppetmasterd --verbose 2>&1"
           run_command_on(cmd, master)
           @prepared = true
         end

data/lib/poolparty/plugins/git.rb

@@ -8,16 +8,16 @@ module PoolParty
       end
             
       def has_git_repos
-        exec({:name => "git-#{name}"}) do
+        has_package(:name => "git-core")
+        exec({:name => "git-#{name}", :requires => package(:name => "git-core")}) do
           command parent.user ? "git clone #{parent.user}@#{parent.source} #{parent.path}" : "git clone #{parent.source} #{parent.to ? parent.to : ""}"
           cwd "#{parent.cwd if parent.cwd}"
           creates "#{::File.join( (parent.cwd ? parent.cwd : cwd), ::File.basename(parent.source, ::File.extname(parent.source)) )}/.git"
-          
+
           exec(:name => "update-#{name}") do
             cwd ::File.dirname(parent.creates)
             command "git pull"
           end
-          
         end
       end
       

data/lib/poolparty/pool/base.rb

@@ -13,7 +13,7 @@ module PoolParty
       :environment => "production",
       :user => "root", # This should change here
       :base_keypair_path => "~/.ec2",
-      :tmp_path => "tmp",
+      :tmp_path => "/tmp/poolparty",
       :remote_storage_path => "/var/poolparty",      
       :fileserver_base => "puppet:///files",
       :base_config_directory => "/etc/poolparty",
@@ -74,13 +74,18 @@ module PoolParty
             "/var/poolparty"           
         ].select do |dir|
           dir if ::File.directory?(dir) && ::File.readable?(dir)
-        end.first || ::File.join(Dir.pwd, "tmp")
+        end.first || ::File.join( "/tmp/poolparty")
       end
       
       def pool_logger_location
         File.join(Dir.pwd, "logs")
       end
       
+      # Array of allowed_commands that you can run on the remote nodes
+      def allowed_commands
+        @allowed_commands ||= open(::File.join( ::File.dirname(__FILE__), "..", "config", "allowed_commands.yml")).read.split(/\n/).map {|a| a.chomp }
+      end
+      
     end
   end    
 end

data/lib/poolparty/pool/cloud.rb

@@ -115,9 +115,7 @@ module PoolParty
           reset_resources!
           add_poolparty_base_requirements
           
-          @build_manifest = returning Array.new do |str|
-
-            str << resources_string_from_resources(resources)
+          @build_manifest = returning Array.new do |str|            
 
             # Refactor this into the resources method
             # TODO
@@ -126,6 +124,8 @@ module PoolParty
               str << @cp.to_string
               str << @cp.include_string
             end
+            
+            str << resources_string_from_resources(resources)
 
             str << "# Custom functions"
             str << Resources::CustomResource.custom_functions_to_string