auser-poolparty 0.2.53 → 0.2.54

data/Manifest.txt

@@ -21,6 +21,7 @@ bin/cloud-ssh
 bin/cloud-start
 bin/cloud-stats
 bin/cloud-terminate
+bin/messenger-get-current-nodes
 bin/pool
 bin/pool-console
 bin/pool-describe
@@ -37,6 +38,9 @@ bin/server-show-stats
 bin/server-start-client
 bin/server-start-master
 bin/server-start-node
+bin/server-stop-client
+bin/server-stop-master
+bin/server-stop-node
 config/hoe.rb
 config/requirements.rb
 examples/basic.rb
@@ -74,8 +78,8 @@ lib/erlang/messenger/ebin/pm_node.beam
 lib/erlang/messenger/ebin/pm_node_rel-0.1.rel
 lib/erlang/messenger/ebin/pm_node_supervisor.beam
 lib/erlang/messenger/ebin/pm_packager.beam
+lib/erlang/messenger/ebin/pm_strings.beam
 lib/erlang/messenger/ebin/utils.beam
-lib/erlang/messenger/erl_crash.dump
 lib/erlang/messenger/include/defines.hrl
 lib/erlang/messenger/lib/eunit/AUTHORS
 lib/erlang/messenger/lib/eunit/CHANGELOG
@@ -154,6 +158,7 @@ lib/erlang/messenger/src/pm_master_supervisor.erl
 lib/erlang/messenger/src/pm_node.erl
 lib/erlang/messenger/src/pm_node_supervisor.erl
 lib/erlang/messenger/src/pm_packager.erl
+lib/erlang/messenger/src/pm_strings.erl
 lib/erlang/messenger/src/utils.erl
 lib/erlang/messenger/useful_snippets
 lib/poolparty.rb
@@ -162,6 +167,7 @@ lib/poolparty/base_packages/haproxy.rb
 lib/poolparty/base_packages/heartbeat.rb
 lib/poolparty/base_packages/poolparty.rb
 lib/poolparty/base_packages/ruby.rb
+lib/poolparty/base_packages/runit.rb
 lib/poolparty/config/allowed_commands.yml
 lib/poolparty/config/postlaunchmessage.txt
 lib/poolparty/core/array.rb
@@ -220,6 +226,7 @@ lib/poolparty/net/remoter.rb
 lib/poolparty/net/remoter_base.rb
 lib/poolparty/plugins/git.rb
 lib/poolparty/plugins/line.rb
+lib/poolparty/plugins/runit.rb
 lib/poolparty/plugins/svn.rb
 lib/poolparty/pool/base.rb
 lib/poolparty/pool/cloud.rb
@@ -232,6 +239,7 @@ lib/poolparty/pool/resource.rb
 lib/poolparty/pool/resources/class_package.rb
 lib/poolparty/pool/resources/conditional.rb
 lib/poolparty/pool/resources/cron.rb
+lib/poolparty/pool/resources/custom_service.rb
 lib/poolparty/pool/resources/directory.rb
 lib/poolparty/pool/resources/exec.rb
 lib/poolparty/pool/resources/file.rb
@@ -251,6 +259,12 @@ lib/poolparty/templates/gem
 lib/poolparty/templates/ha.cf
 lib/poolparty/templates/haproxy.conf
 lib/poolparty/templates/haresources
+lib/poolparty/templates/messenger/client/log-run.erb
+lib/poolparty/templates/messenger/client/run.erb
+lib/poolparty/templates/messenger/master/log-run.erb
+lib/poolparty/templates/messenger/master/run.erb
+lib/poolparty/templates/messenger/node/log-run.erb
+lib/poolparty/templates/messenger/node/run.erb
 lib/poolparty/templates/namespaceauth.conf
 lib/poolparty/templates/poolparty.monitor
 lib/poolparty/templates/puppet.conf

data/README.txt

@@ -4,18 +4,30 @@ http://poolpartyrb.com
 
 == DESCRIPTION:
 
-COMING SOON
+PoolParty makes it easy and simple to configure any cloud of computers. In clear language, describe your cloud
+with language such as:
+
+pool :cloud do
+ cloud :app do
+  apache do
+   has_virtualhost(:name => "/var/www/sites/poolpartyrb.com")
+  end
+ end
+end
 
 == FEATURES/PROBLEMS:
 
-COMING SOON
+* Written in Ruby and Erlang
+* Written from the ground up to be extensible with plugins
+* Easy git-style commands to communicate with your clouds
+* Much much more
 
 == SYNOPSIS:
 
-poolparty is written with the intention of being as application-agnostic as possible. It installs only the basic 
+PoolParty is written with the intention of being as application-agnostic as possible. It installs only the basic 
 required software to glue the cloud together on the instances as listed below.
 
-poolparty is easily configuration. In fact, it makes little assumptions about your development environment and allows 
+PoolParty is easily configuration. In fact, it makes little assumptions about your development environment and allows 
 several options on how to begin configuring the cloud.
 
 == REQUIREMENTS:
@@ -26,6 +38,10 @@ COMING SOON
 
 sudo gem install auser-poolparty
 
+== TODO:
+* Replace services with Runit
+* Refactor provisioning to use erlang
+
 == LICENSE:
 
 (The MIT License)

data/bin/cloud-ensure-provisioning

@@ -8,17 +8,23 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n name', '--name name', 'Host name')    { |h| optioner.hostname h }
   opts.on('-l', '--no-shell', 'No shell') {optioner.noshell true}
 end
-@hostname = o.hostname ? o.hostname : `hostname`.chomp
-@hostname = "node0" if @hostname == "master" # Quick fix to make sure we have a node running on the master as well
 
 o.loaded_clouds.each do |cloud|
   
-  with_cloud(cloud, {:hostname => @hostname}) do
+  with_cloud(cloud) do
     
-    boot_file = "#{Messenger.append_dir}/pm_node_rel-0.1"
-    command = Messenger.erl_command(hostname, "-boot #{boot_file} #{noshell ? "" : "-detached -heart"}")
-    
-    Kernel.system command
+    @nonprovisioned_nodes = list_of_running_instances.map {|a| a.name } - cloud.get_current_nodes
+    # @tp = ThreadPool.new(10)
+    @nonprovisioned_nodes.each do |node|
+      vputs "Provisioning #{node}"
+      next if node == "master"
+      # @tp.process do
+        PoolParty::Provisioner.process_clean_reconfigure_for!(node, cloud)
+        `cloud-provision -i #{node.gsub(/node/, '')}`
+        PoolParty::Provisioner.process_clean_reconfigure_for!(node, cloud)
+      # end
+    end
+    # @tp.join
   end  
   
 end

data/bin/cloud-handle-load

@@ -11,6 +11,7 @@ o.loaded_clouds.each do |cloud|
 
   with_cloud(cloud) do
     vputs header("Load handling cloud #{name}")
+    vputs "should_expand_cloud: #{should_contract_cloud?}"
     if should_expand_cloud?
       vputs "Expanding cloud based on load"
       logger.debug "Expanding cloud based on load"

data/bin/cloud-start

@@ -22,7 +22,7 @@ o.loaded_clouds.each do |cloud|
         provisioning_complete
       end
     end
-    vputs open(::File.join(File.dirname(__FILE__), "..", "lib", "poolparty", "config", "postlaunchmessage.txt")).read ^ {:master_ip => master.ip.chomp}
+    puts open(::File.join(File.dirname(__FILE__), "..", "lib", "poolparty", "config", "postlaunchmessage.txt")).read ^ {:master_ip => master.ip.chomp}
     clear_base_directory unless testing
   end
   

data/bin/messenger-get-current-nodes

@@ -0,0 +1,13 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolpartycl"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+end
+
+o.loaded_clouds.each do |cloud|
+  # @nodes = cloud.messenger_send!("get_current_nodes")
+  # @nodes = @nodes.split(" ").map {|a| a.split(/@/)[-1] }
+  puts cloud.get_current_nodes.join(" ")
+end

data/bin/server-show-stats

@@ -8,7 +8,7 @@ end
 
 o.loaded_clouds.each do |cloud|
   
-  with_cloud(cloud, {:testing => o.testing }) do    
+  with_cloud(cloud) do    
     puts header("Stats")
     puts rules_values
   end  

data/bin/server-start-client

@@ -10,13 +10,20 @@ end
 o.loaded_clouds.each do |cloud|
   
   with_cloud(cloud) do
-    
-    boot_file = "#{Messenger.append_dir}/pm_client_rel-0.1"
-    command = Messenger.erl_command("client", "-boot #{boot_file} #{noshell ? "" : "-detached -heart -noshell"}", 7050, 7050)
-    
-    vputs "Running #{command}"
-    
-    Kernel.system "#{command}" unless testing
+        
+    # ruby /var/lib/gems/1.8/bin/server-start-client    
+    already_running = %x[ps aux | grep beam | grep -v grep | grep client]
+    if already_running.chomp.empty?
+      
+      boot_file = "#{Messenger.append_dir}/pm_client_rel-0.1"
+
+      Kernel.system "cd #{Messenger.append_dir} && rake build_boot_scripts"  unless ::File.file?("#{boot_file}.boot") || testing    
+      command = Messenger.erl_command("client", "-boot #{boot_file} #{noshell ? "" : "-detached -heart -noshell"}", 7049, 7050)
+      vputs "Running #{command}"
+      
+      Kernel.system "export HOME=/root && #{command}" unless testing
+    end
   end  
   
-end
+end
+

data/bin/server-start-master

@@ -10,13 +10,17 @@ end
 o.loaded_clouds.each do |cloud|
   
   with_cloud(cloud) do
-    
-    boot_file = "#{Messenger.append_dir}/pm_master_rel-0.1"
-    command = Messenger.erl_command("master", "-boot #{boot_file} #{noshell ? "" : "-noshell -detached -heart"}")
-    
-    vputs "Running #{command}"
-    
-    Kernel.system "#{command}" unless testing
+            
+    already_running = %x[ps aux | grep beam | grep -v grep | grep master]        
+    if already_running.chomp.empty?    
+      
+      boot_file = "#{Messenger.append_dir}/pm_master_rel-0.1"
+      Kernel.system ". /etc/profile && server-build-messenger"  unless ::File.file?("#{boot_file}.boot") || testing    
+      command = Messenger.erl_command("master", "-boot #{boot_file} #{noshell ? "" : "-noshell -detached -heart"}")    
+      vputs "Running #{command}"
+      
+      Kernel.system "export HOME=/root && #{command}" unless testing
+    end
   end  
   
 end

data/bin/server-start-node

@@ -15,16 +15,18 @@ available_monitors = PoolParty::Monitors.available_monitors
 
 o.loaded_clouds.each do |cloud|
   
-  with_cloud(cloud) do
-    # TODO: Change this to be app specfic
-    # SECURITY RISK
-    
-    boot_file = "#{Messenger.append_dir}/pm_node_rel-0.1"
-    command = Messenger.erl_command(hostname, "-boot #{boot_file} #{noshell ? "" : "-noshell -detached -heart"} -- #{available_monitors.join(" ")}")
-    
-    vputs "Running #{command}"
-    
-    Kernel.system "#{command}" unless testing
+  with_cloud(cloud, {:hostname => @hostname}) do
+    # TODO: Change this to be app specfic    
+    already_running = %x[ps aux | grep beam | grep -v grep | grep node]
+    if already_running.chomp.empty?    
+      boot_file = "#{Messenger.append_dir}/pm_node_rel-0.1"
+
+      Kernel.system ". /etc/profile && server-build-messenger"  unless ::File.file?("#{boot_file}.boot") || testing
+      command = Messenger.erl_command(hostname, "-boot #{boot_file} #{noshell ? "" : "-noshell -detached -heart"} -- #{available_monitors.join(" ")}")    
+      vputs "Running #{command}"
+      
+      Kernel.system "export HOME=/root && #{command}" unless testing
+    end
   end  
   
 end

data/bin/server-stop-client

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ps aux | grep -v grep | grep client_service | awk '{print $2}' | xargs kill

data/bin/server-stop-master

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ps aux | grep -v grep | grep pm_master | awk '{print $2}' | xargs kill

data/bin/server-stop-node

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+ps aux | grep -v grep | grep pm_node | awk '{print $2}' | xargs kill

data/lib/erlang/messenger/include/defines.hrl

@@ -17,6 +17,7 @@
 -define (MASTER_NODE_NAME, master).
 -define (MASTER_SERVER, global:whereis_name(pm_master)).
 
+-define (UPDATE_TIME, 10000).
 -define(DICT, dict).
 -record (node, 
 					{load}).

data/lib/erlang/messenger/pm_client_rel-0.1.script

@@ -1,4 +1,4 @@
-%% script generated at {2008,11,10} {16,32,20}
+%% script generated at {2008,11,11} {20,31,50}
 {script,
     {"client","0.1"},
     [{preLoaded,

data/lib/erlang/messenger/pm_master_rel-0.1.script

@@ -1,4 +1,4 @@
-%% script generated at {2008,11,10} {16,32,20}
+%% script generated at {2008,11,11} {20,31,50}
 {script,
     {"master","0.1"},
     [{preLoaded,

data/lib/erlang/messenger/pm_node_rel-0.1.script

@@ -1,4 +1,4 @@
-%% script generated at {2008,11,10} {16,32,20}
+%% script generated at {2008,11,11} {20,31,50}
 {script,
     {"node","0.1"},
     [{preLoaded,

data/lib/erlang/messenger/src/client_server.erl

@@ -5,34 +5,48 @@
 
 -define (RECONNECT_TIMEOUT, 10000).
 
-start() -> 
-	?TRACE("MASTER_LOCATION", [?MASTER_LOCATION]),
-	connect_to_master(),
-	global:sync(),
+start() -> 	
+	utils:start_timer(client_timer, ?UPDATE_TIME, fun() -> client_server:connect_to_master() end),
 	pm_client:start(?MODULE, 7050, {?MODULE, loop}).
 	
 master_server() -> global:whereis_name(pm_master).
 
 loop(Socket) ->
     case gen_tcp:recv(Socket, 0) of
-        {ok, Data} ->
+        {ok, Data} ->						
 						?TRACE("received", [master_server(), erlang:binary_to_list(Data)]),
 						% Args = [Item || K <- string:tokens(erlang:binary_to_list(Data), " "), Item <- erlang:list_to_atom(K)],
 						[Meth|Args] = string:tokens(erlang:binary_to_list(Data), " "),
 						?TRACE("received", [Meth, Args]),
 						Output = gen_server:call(master_server(), {erlang:list_to_atom(Meth), Args}),
 						?TRACE("received from gen_server", [Output]),
-            gen_tcp:send(Socket, erlang:float_to_list(Output)),
-						io:format("~p~n", [Output]),
+						send_back_appropriate_response(Socket, Output),
 						?TRACE("posted", [Output]),
             loop(Socket);
         {error, closed} ->
             ok
     end.
 
+% send_back_appropriate_response(Socket, Output) when is_float(Output) -> gen_tcp:send(Socket, erlang:float_to_list(Output));
+% Figure out how to do this the best... damnit
+send_back_appropriate_response(Socket, Output) -> 
+	[H|_T] = Output,
+	case erlang:is_atom(H) of
+		true ->
+			?TRACE("NewOut", [Output]),
+			NewOut = pm_strings:string_join( [erlang:atom_to_list(K) || K <- Output], " ");
+		_ ->
+			ListOfFloats = pm_strings:string_join( [erlang:float_to_list(V) || V <- Output], " "),
+			NewOut = [ListOfFloats]
+	end,
+	?TRACE("NewOut", [NewOut]),
+	gen_tcp:send(Socket, NewOut).
+	
+
 connect_to_master() ->
 	case net_adm:ping(?MASTER_LOCATION) of
-		pong ->
+		pong ->			
+			global:sync(),
 			ok;
 		_ ->
 			receive 

data/lib/erlang/messenger/src/pm_master.erl

@@ -44,8 +44,7 @@
 get_load(Type) ->
 	% {Loads, _} = pm_cluster:send_call(get_load_for_type, [Type]),
 	% {Loads, _} = gen_server:call(?SERVER, {get_load, [Type]}),
-	Loads = gen_server:call(?SERVER, {get_current_load, Type}),
-	utils:average_of_list(Loads).
+	gen_server:call(?SERVER, {get_current_load, Type}).
 
 % Send reconfigure tasks to every node
 reconfigure_cloud() -> gen_server:cast(?SERVER, {force_reconfig}).
@@ -59,7 +58,7 @@ shutdown_cloud() ->
 	pm_cluster:send_call(stop, []),
 	{ok}.
 
-get_current_nodes() -> gen_server:call(?SERVER, {get_live_nodes}).
+get_current_nodes() -> gen_server:call(?SERVER, {get_current_nodes, []}).
 
 stop() -> gen_server:cast(?MODULE, stop).	
 %%--------------------------------------------------------------------
@@ -101,13 +100,12 @@ handle_call({get_load, Args}, _From, State) ->
 		Nodes = pm_cluster:get_live_nodes(),
 		List = rpc:multicall(Nodes, pm_node, get_load_for_type, [Args]),
 		Loads = utils:convert_responses_to_int_list(List),
-		{reply, Loads, State};		
-handle_call({get_current_load, [Type]}, _From, State) ->
-	LoadForType = get_load_for_type(Type, State),
+		{reply, Loads, State};
+handle_call({get_current_load, Types}, _From, State) ->	
+	LoadForType = [utils:average_of_list(get_load_for_type(Type, State)) || Type <- Types],
 	?TRACE("LoadForType: ",[LoadForType]),
-	Loads = utils:average_of_list(LoadForType),
-	{reply, Loads, State};
-handle_call({get_live_nodes}, _From, State) ->
+	{reply, LoadForType, State};
+handle_call({get_current_nodes, _Args}, _From, State) ->
 	{reply, get_live_nodes(State), State}.
 	
 % handle_call(_Request, _From, State) ->

data/lib/erlang/messenger/src/pm_node.erl

@@ -21,7 +21,6 @@
 					monitors = {}  % Tuple of monitors
 				}).
 -define(SERVER, ?MODULE).
--define (UPDATE_TIME, 10000).
 
 % Client function definitions
 -export ([stop/0]).

data/lib/erlang/messenger/src/pm_strings.erl

@@ -0,0 +1,11 @@
+-module (pm_strings).
+-include_lib("../include/defines.hrl").
+-compile(export_all).
+
+string_join(Items, Sep) ->
+    lists:flatten(lists:reverse(string_join1(Items, Sep, []))).
+
+string_join1([Head | []], _Sep, Acc) ->
+    [Head | Acc];
+string_join1([Head | Tail], Sep, Acc) ->
+    string_join1(Tail, Sep, [Sep, Head | Acc]).

data/lib/erlang/messenger/src/utils.erl

@@ -1,4 +1,5 @@
 -module (utils).
+-include_lib("../include/defines.hrl").
 -compile(export_all).
 
 -ifdef(EUNIT).
@@ -12,18 +13,25 @@ convert_responses_to_int_list(L) ->
 	average_for_list(Sum, L).
 
 % Start a timer to fire off Fun after Time number of milliseconds
+start_timer(Name, Time, Fun) ->
+	case whereis(Name) of
+		undefined -> register(Name, spawn(fun() -> tick_timer(Time, Fun) end));
+		_ -> ok
+	end.
 start_timer(Time, Fun) -> 
-	register(?MODULE, spawn(fun() -> tick_timer(Time, Fun) end)). 
+	start_timer(?MODULE, Time, Fun).
+	% register(?MODULE, spawn(fun() -> tick_timer(Time, Fun) end)). 
 
+stop_timer(Name) -> erlang:whereis(Name) ! stop.
 stop_timer() -> ?MODULE ! stop. 
 
+
 tick_timer(Time, Fun) -> 
 	receive 
-		stop -> 
-			void 
+		stop -> void 
 	after Time -> 
 		Fun(), 
-		tick_timer(Time, Fun) 
+		tick_timer(Time, Fun)
 	end. 
 
 average_of_list(L) ->