poolparty 0.2.18 → 0.2.69

data/lib/erlang/messenger/src/pm_node.erl

@@ -8,22 +8,89 @@
 -module(pm_node).
 -behaviour(gen_server).
 
+-include_lib("../include/defines.hrl").
+
 %% API
--export([start_link/0]).
+-export([start_link/1, start_link/0]).
 
 %% gen_server callbacks
 -export([init/1, handle_call/3, handle_cast/2, handle_info/2,
          terminate/2, code_change/3]).
--record(state, {}).
+
+-record(state, {
+					monitors = {},  % Tuple of monitors
+					stored_loads = ?DICT:new() % Dictionary of latest loads
+				}).
 -define(SERVER, ?MODULE).
 
 % Client function definitions
 -export ([stop/0]).
--export ([get_load_for_type/1, fire_cmd/1, run_reconfig/0]).
-
+-export ([get_current_load_for_type/1, run_cmd/1, fire_cmd/1]).
+-export ([run_reconfig/0, local_update/1, still_here/0, print_monitors/0]).
+-export ([server_location/0]).
 %%====================================================================
 %% API
 %%====================================================================
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+
+
+% Get the load for the type sent...
+get_current_load_for_type(Type) ->
+	String = string:concat(". /etc/profile && server-get-load -m ",Type),
+	LatestLoad = os:cmd(String), 
+	Load = gen_server:call(server_location(), {update_load, Type, LatestLoad}),
+	{Load}.
+
+get_load_listing(Type, State) ->
+	% Find or create the stored node
+	case ?DICT:is_key(Type, State#state.stored_loads) of
+		true -> 
+			LoadStore = ?DICT:fetch(Type, State#state.stored_loads), 
+			{LoadStore, State};
+		false -> 
+			LoadStore = [],
+			NewState = State#state{stored_loads = ?DICT:store(Type, LoadStore, State#state.stored_loads)}, 
+			{LoadStore, NewState}
+	end.
+
+get_average_load_listing(Type, State) ->
+	{LoadListing, _} = get_load_listing(Type, State),
+	utils:average_of_list(LoadListing).
+% Get the stored load for the type
+% Only store the latest NUM_LOADS_TO_STORE in the stored_loads array
+store_load_for_type(Type, Load, State) ->
+	{LoadStore, LoadListingState} = get_load_listing(Type, State),	
+	LoadList = lists:reverse(LoadStore),
+	case length(LoadList) >= ?NUM_LOADS_TO_STORE of
+		true -> [_|T] = LoadList; 
+		false -> T = LoadList
+	end,
+	NewStoredLoad = lists:append([Load], lists:reverse(T)),	
+	NewState = State#state{stored_loads = ?DICT:store(Type, NewStoredLoad, LoadListingState#state.stored_loads)},	
+	{Type, NewState}.
+	
+% Rerun the configuration
+run_reconfig() -> gen_server:cast(server_location(), {run_reconfig}).
+print_monitors() -> gen_server:call(server_location(), {print_monitors}).
+
+% Allows us to fire off any command (allowed by poolparty on the check)
+run_cmd(Cmd) -> gen_server:call(server_location(), {run_command, Cmd}).
+fire_cmd(Cmd) -> gen_server:cast(server_location(), {fire_command, Cmd}).
+
+still_here() -> gen_server:call(server_location(), {still_there}).
+
+% Stop the pm_node entirely
+stop() -> gen_server:cast(server_location(), stop).
+
+% Run every UPDATE_TIME seconds
+local_update(Types) ->
+	?TRACE("Updating", [?MASTER_LOCATION]),	
+	net_adm:ping(?MASTER_LOCATION), % check in with the master
+	Load = [{Ty, element(1, get_current_load_for_type(Ty))} || Ty <- Types],
+	gen_server:cast(?MASTER_SERVER, {update_node_load, node(), Load}).	
+	
 %%--------------------------------------------------------------------
 %% Function: start_link() -> {ok,Pid} | ignore | {error,Error}
 %% Description: Starts the server
@@ -33,9 +100,8 @@
 %% 
 %% Fires a ping every 10 seconds
 %%--------------------------------------------------------------------
-start_link() ->
-	utils:start_timer(10000, fun() -> net_adm:ping(master) end),
-  gen_server:start_link({global, ?SERVER}, ?MODULE, [], []).
+start_link() -> start_link(init:get_plain_arguments()).
+start_link(Args) -> gen_server:start_link({global, node()}, ?MODULE, Args, Args).
 
 %%====================================================================
 %% gen_server callbacks
@@ -48,9 +114,14 @@ start_link() ->
 %%                         {stop, Reason}
 %% Description: Initiates the server
 %%--------------------------------------------------------------------
-init([]) ->
+init(Args) ->
+	io:format("Master location ~p~n", [?MASTER_LOCATION]),
 	process_flag(trap_exit, true),
-  {ok, #state{}}.
+	utils:start_timer(?UPDATE_TIME, fun() -> pm_node:local_update(Args) end),
+  {ok, #state{
+					monitors=Args,
+					stored_loads = ?DICT:new()
+				}}.
 
 %%--------------------------------------------------------------------
 %% Function: %% handle_call(Request, From, State) -> {reply, Reply, State} |
@@ -61,6 +132,21 @@ init([]) ->
 %%                                      {stop, Reason, State}
 %% Description: Handling call messages
 %%--------------------------------------------------------------------
+handle_call({run_command, Cmd}, _From, State) ->
+	Reply = os:cmd(". /etc/profile && server-fire-cmd \""++Cmd++"\""),
+	{reply, Reply, State};
+handle_call({update_load, Type, Load}, _From, State) ->
+	{Type, NewState} = store_load_for_type(Type, Load, State),
+	Reply = get_average_load_listing(Type, NewState),
+	?TRACE("Updated load", [Reply]),
+	{reply, Reply, NewState};
+handle_call({still_there}, _From, State) ->
+	Reply = still_here,
+	{reply, Reply, State};
+handle_call({print_monitors}, _From, State) ->
+	[ io:format("Type: ~p ", [Monitor]) || Monitor <- State#state.monitors ],
+	io:format("~n"),
+	{reply, ok, State};
 handle_call(_Request, _From, State) ->
   Reply = ok,
   {reply, Reply, State}.
@@ -71,8 +157,16 @@ handle_call(_Request, _From, State) ->
 %%                                      {stop, Reason, State}
 %% Description: Handling cast messages
 %%--------------------------------------------------------------------
+handle_cast({fire_command, Cmd}, State) ->
+	?TRACE("Running command: ~p~n", [Cmd]),
+	os:cmd(". /etc/profile && server-fire-cmd \""++Cmd++"\" 2>&1 > /dev/null"),
+	{noreply, State};
+handle_cast({run_reconfig}, State) ->
+	?TRACE("Running Reconfig", ["server-rerun"]),
+	os:cmd(". /etc/profile && server-rerun"),
+	{noreply, State};
 handle_cast(_Msg, State) ->
-  {noreply, State}.
+	{noreply, State}.
 
 %%--------------------------------------------------------------------
 %% Function: handle_info(Info, State) -> {noreply, State} |
@@ -102,23 +196,6 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
   {ok, State}.
 
-%%--------------------------------------------------------------------
-%%% Internal functions
-%%--------------------------------------------------------------------
-
-% Get the load for the type sent...
-get_load_for_type(Type) ->
-	String = string:concat(". /etc/profile && server-get-load -m ",Type),
-	{os:cmd(String)}.
-
-% Rerun the configuration
-run_reconfig() ->	{os:cmd(". /etc/profile && server-rerun")}.
-
-% Allows us to fire off any command (allowed by poolparty on the check)
-fire_cmd(Cmd) ->
-	String = ". /etc/profile && server-fire-cmd \""++Cmd++"\"",
-	{os:cmd(String)}.
-
-% Stop the pm_node entirely
-stop() ->
-	gen_server:cast(?MODULE, stop).
+% Private
+server_location() ->
+	global:whereis_name(node()).

data/lib/erlang/messenger/src/pm_node_supervisor.erl

@@ -4,16 +4,16 @@
 -module (pm_node_supervisor).
 -behaviour(supervisor).
 
--export([start/0, start_in_shell_for_testing/0, start_link/1, init/1]).
+-export ([start/0, start/1]).
+-export([start_in_shell_for_testing/0, start_link/1, init/1]).
 
 -ifdef(EUNIT).
 -include_lib("eunit/include/eunit.hrl").
 -endif.
 
-start() ->
-	spawn(fun() ->
-			supervisor:start_link({local, ?MODULE}, ?MODULE, _Arg = [])
-		end).
+start() -> spawn(fun() -> supervisor:start_link({local, ?MODULE}, ?MODULE, _Arg = []) end).
+start(Args) -> spawn(fun() -> supervisor:start_link({local, ?MODULE}, ?MODULE, Args) end).
+	
 
 start_in_shell_for_testing() ->
 	{ok, Pid} = supervisor:start_link({local, ?MODULE}, ?MODULE, _Arg = []),
@@ -22,19 +22,17 @@ start_in_shell_for_testing() ->
 start_link(Args) ->
 	supervisor:start_link({local, ?MODULE}, ?MODULE, Args).
 
-init([]) ->
+init(Args) ->
 	RestartStrategy = one_for_one,
-	MaxRestarts = 3, % 1000
-	MaxTimeBetRestarts = 30, % 3600
+	MaxRestarts = 1000,
+	MaxTimeBetRestarts = 3600,
+	TimeoutTime = 5000,
 
 	SupFlags = {RestartStrategy, MaxRestarts, MaxTimeBetRestarts},
+	
+	EventManager = {pm_event_manager,  {pm_event_manager, start_link, []}, permanent, TimeoutTime, worker, dynamic},
+	NodeServer = {pm_node1, {pm_node, start_link, Args}, permanent, TimeoutTime, worker, [pm_node]},
 
-	LoadServers = [
-		{pm_node1,
-			{pm_node, start_link, []}, 
-			permanent, 5000, worker, 
-			[pm_node]
-		}
-	],
+	LoadServers = [EventManager, NodeServer],
 
 	{ok, {SupFlags, LoadServers}}.

data/lib/erlang/messenger/src/pm_packager.erl

@@ -20,7 +20,7 @@ start() ->
 %% OTP design principles as a supervision tree, this means starting the
 %% top supervisor of the tree.
 %%--------------------------------------------------------------------
-start(_Type, _) ->
+start(_, _) ->
 	recompile_scripts("0.1").
 
 %%--------------------------------------------------------------------
@@ -52,11 +52,14 @@ gen_rel(Name, Vers) ->
 recompile_scripts(Vers) ->
 	gen_rel("master", Vers),
 	gen_rel("node", Vers),
+	gen_rel("client", Vers),
 	systools:make_script("pm_node_rel-"++Vers, [local,{path,["ebin"]}]),
-	systools:make_script("pm_master_rel-"++Vers, [local,{path,["ebin"]}]).
+	systools:make_script("pm_master_rel-"++Vers, [local,{path,["ebin"]}]),
+	systools:make_script("pm_client_rel-"++Vers, [local,{path,["ebin"]}]).
 
 package_scripts(Vers) ->
 	systools:make_tar("ebin/pm_node_rel-"++Vers),
+	systools:make_tar("ebin/pm_client_rel-"++Vers),
 	systools:make_tar("ebin/pm_master_rel-"++Vers).
 
 get_vsn(Module) ->

data/lib/erlang/messenger/src/pm_strings.erl

@@ -0,0 +1,11 @@
+-module (pm_strings).
+-include_lib("../include/defines.hrl").
+-compile(export_all).
+
+string_join(Items, Sep) ->
+    lists:flatten(lists:reverse(string_join1(Items, Sep, []))).
+
+string_join1([Head | []], _Sep, Acc) ->
+    [Head | Acc];
+string_join1([Head | Tail], Sep, Acc) ->
+    string_join1(Tail, Sep, [Sep, Head | Acc]).

data/lib/erlang/messenger/src/utils.erl

@@ -1,4 +1,5 @@
 -module (utils).
+-include_lib("../include/defines.hrl").
 -compile(export_all).
 
 -ifdef(EUNIT).
@@ -12,27 +13,61 @@ convert_responses_to_int_list(L) ->
 	average_for_list(Sum, L).
 
 % Start a timer to fire off Fun after Time number of milliseconds
+start_timer(Name, Time, Fun) ->
+	case whereis(Name) of
+		undefined -> register(Name, spawn(fun() -> tick_timer(Time, Fun) end));
+		_ -> ok
+	end.
 start_timer(Time, Fun) -> 
-	register(?MODULE, spawn(fun() -> tick_timer(Time, Fun) end)). 
+	start_timer(?MODULE, Time, Fun).
+	% register(?MODULE, spawn(fun() -> tick_timer(Time, Fun) end)). 
 
+stop_timer(Name) -> erlang:whereis(Name) ! stop.
 stop_timer() -> ?MODULE ! stop. 
 
+
 tick_timer(Time, Fun) -> 
 	receive 
-		stop -> 
-			void 
+		stop -> void 
 	after Time -> 
 		Fun(), 
-		tick_timer(Time, Fun) 
-end. 
+		tick_timer(Time, Fun)
+	end. 
 
+average_of_list(L) ->
+	Sum = lists:foldr( fun(Int, Sum) -> 
+		case erlang:is_float(Int) of
+			true -> Integer = Int;
+			false -> Integer = erlang:list_to_float(Int)
+		end,
+		Integer + Sum 
+	end, 0, [F || F <- L] ),
+	average_for_list(Sum, L).
 
-% Private
 % Get the average of the list
 average_for_list(Num, L) ->
 	case length(L) of
 		0 ->
-			0;
+			-1;
 		_ ->
 			Num / length(L)
-	end.
+	end.
+
+
+
+% Provisioning utils
+distribute_modules_to(Modules, Nodes) ->
+	% transfer the modules to all the nodes
+	io:format("Sending ~p to ~p~n", [Modules, Nodes]),
+	lists:foreach(fun(Node) ->
+			transfer_modules(Node, Modules)
+	end, Nodes).
+
+% Transfer modules of code to this node
+transfer_modules(Node, Modules) ->
+	[transfer_module(Node, M) || M <- Modules].
+
+% Transfer one module to the Node
+transfer_module(Node, Module) ->
+	{_Module, Binary, Filename} = code:get_object_code(Module),
+	rpc:call(Node, code, load_binary, [Module, Filename, Binary]).

data/lib/erlang/messenger/useful_snippets

@@ -0,0 +1,17 @@
+% Snippets
+% lists:map(fun(No) -> net_adm:ping(No) end, pm_cluster:any_new_servers()).
+
+% Start test nodes, defining testing
+% erl -pa ./ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -setcookie poolparty -boot pm_node_rel-0.1 -sname node0
+% erl -pa ./ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -setcookie poolparty -boot pm_node_rel-0.1 -sname node1
+
+% Starting the client on the master
+% erl -pa /var/lib/gems/1.8/gems/poolparty-0.2.54/lib/erlang/messenger/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname client -setcookie poolparty -run pm_client start
+
+% Get the load
+% erl -pa /var/lib/gems/1.8/gems/poolparty-0.2.46/lib/erlang/messenger/ebin -kernel inet_dist_listen_min 7000 inet_dist_listen_max 7050 -sname client -setcookie poolparty -run pm_client start
+
+% cd /var/poolparty && gem uninstall poolparty && wget http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=true -O poolparty.gem 2>&1 && gem install --no-ri --no-rdoc poolparty.gem
+
+% LOAD TESTING
+% ab -n 10000 -c 5 http://75.101.162.232/

data/lib/poolparty/aska/aska.rb

@@ -0,0 +1,150 @@
+=begin rdoc
+  Aska
+  TODO: Extract this into a proper gem
+=end
+module Aska
+  module ClassMethods
+    def rules(name=:rules, arr=[], create_vars=true)
+      returning look_up_rules(name) do |rs|
+        arr.each do |line|
+          next unless line
+          rule = Rule.new(line)
+          next unless rule.valid?
+          k = rule.key
+          v = rule.var
+          m = rule.comparison
+
+          create_instance_variable(k, create_vars)
+          rs << {k => [m, v]}
+          rs << {k => [">", "0"]} unless rs.reject {|a| a.to_s == "#{k}>0" }
+        end
+        self.send(:define_method, name) do
+          look_up_rules(name)
+        end        
+      end
+    end
+    def create_instance_variable(k, create_vars=true)
+      aska_attr_accessors << k.to_sym unless aska_attr_accessors.include?(k.to_sym)
+      if create_vars
+        attr_reader k.to_sym unless respond_to?("#{k}".to_sym)
+        attr_writer k.to_sym unless respond_to?("#{k}=".to_sym)
+      end
+    end
+    def look_up_rules(name)
+      defined_rules[name.to_sym] ||= Rules.new
+    end
+    def are_rules?(name)
+      !look_up_rules(name).empty?
+    end
+    def aska_attr_accessors
+      @aska_attr_accessors ||= Rules.new
+    end
+    def defined_rules
+      @defined_rules ||= {}
+    end
+    def aska_named(name)
+      "#{name}_aska"
+    end
+  end
+  
+  module InstanceMethods
+    def rules
+      @rules ||= self.class.defined_rules
+    end
+    def valid_rules?(name=:rules)
+      self.class.look_up_rules(name).reject {|rule| valid_rule?(rule) }.empty?
+    end
+    def __aska_send_value(m)
+      if respond_to?(m.to_sym)
+        self.send(m.to_sym)
+      else
+        m
+      end
+    end
+    def valid_rule?(rule)
+      rule.each do |key,value|
+        begin          
+          return __aska_send_value(key).send(value[0].to_sym, __aska_get_var(value[1]))
+        rescue Exception => e
+          return false
+        end
+      end
+    end
+    def rules_values
+      returning Array.new do |arr|
+        self.class.defined_rules.each do |name,rule_array|
+          arr << "#{name} : #{valid_rules?(name.to_sym)}"
+          rule_array.each do |rule|
+            rule.map do |k,v|
+              arr << "  #{k} -> #{__aska_send_value(k)} (#{v[0]} #{v[1]})"
+            end
+          end
+        end
+      end.join("\n")
+    end
+    # Get the variable from the class
+    # If it's defined as an attr_accessor, we know it has been defined as a rule
+    # Otherwise, if we are passing it as a 
+    def __aska_get_var(name)
+      # attr_accessor?(name) ? aska(name) : 
+      (supported_method?(name) ? name.to_sym : name.to_f)
+    end
+    def __aska_aska(name)
+      self.class.aska_named(name)
+    end
+    def attr_accessor?(name)
+      self.class.aska_attr_accessors.include?(name.to_sym)
+    end
+    def supported_method?(meth)
+      %w(< > == => =<).include?("#{meth}")
+    end
+    
+    def look_up_rules(r);self.class.look_up_rules(r);end
+    def are_rules?(r);self.class.are_rules?(r);end
+    
+    # def method_missing(m, *args, &block)
+    #   if self.class.defined_rules.has_key?(m.to_sym)
+    #     self.class.send(:define_method, m) do
+    #       self.class.look_up_rules(m)
+    #     end
+    #     self.send m
+    #   else
+    #     super
+    #   end
+    # end
+  end
+  
+  def self.included(receiver)
+    receiver.extend         ClassMethods
+    receiver.send :include, InstanceMethods
+  end
+  
+  class Rule < String
+    attr_accessor :value
+    
+    def initialize(v)
+      @value = v
+    end
+    def valid?
+      value =~ /(.+)[=\\\<\>](.*)/
+    end
+    def key
+      value[/(.+)[=\\\<\>](.*)/, 1].gsub(/\s+/, '')
+    end
+    def comparison
+      value[/[=\\<>]/, 0].gsub(/\s+/, '')
+    end
+    def var
+      value[/(.+)[=\\<>](.*)/, 2].gsub(/\s+/, '')
+    end
+    def value
+      @value ||= ""
+    end
+  end
+  
+  class Rules < Array
+    def to_s
+      self.map {|r| v=r.keys.first;"'#{v} #{r[v][0]} #{r[v][1]}'"}.join(", ")
+    end
+  end
+end

data/lib/poolparty/base_packages/haproxy.rb

@@ -3,43 +3,36 @@ module PoolParty
     plugin :haproxy do
       
       def enable
-        execute_if("$hostname", "master") do
-          package({:name => "haproxy"})
+        execute_on_master do
+          has_package({:name => "haproxy"})
 
+          # Restart sysklogd after we update the haproxy.log
+          has_service(:name => "sysklogd")
+          
+          # Template variables          
+          has_variable(:name => "name_haproxy", :value => "#{cloud.name}")
+          has_variable(:name => "nodenames_haproxy", :value => "generate('/usr/bin/env', '/usr/bin/server-list-active', '-c', 'name', '-n', '#{cloud.name}')")
+          has_variable(:name => "node_ips_haproxy",  :value => "generate('/usr/bin/env', '/usr/bin/server-list-active', '-c', 'ip', '-n', '#{cloud.name}')")
+          
+          has_variable(:name => "ports_haproxy", :value => ([(self.respond_to?(:port) ? port : Base.port)].flatten))        
+          has_variable(:name => "forwarding_port", :value => (respond_to?(:forwarding_port) ? forwarding_port : Base.forwarding_port))
+          has_variable(:name => "proxy_mode", :value => (respond_to?(:proxy_mode) ? proxy_mode : Base.proxy_mode))
+          
           # Startup haproxy and enable it
           has_line_in_file("ENABLED=1", "/etc/default/haproxy")
           has_line_in_file("SYSLOGD=\"-r\"", "/etc/default/syslogd")
-          has_line_in_file("local0.* /var/log/haproxy.log", "/etc/syslog.conf", {:notify => 'Service["sysklogd"]'})
-
-          # Restart sysklogd after we update the haproxy.log
-          has_service(:name => "sysklogd") do
-            ensures "running"
-          end
+          has_line_in_file("local0.* /var/log/haproxy.log", "/etc/syslog.conf", {:notify => get_service("sysklogd")})
 
           # Service is required
-          has_service(:name => "haproxy")
-
-          # Tempalte variables
-          variable(:name => "name_haproxy", :value => "#{@parent.name}")
-          
-          if @parent.provisioning?
-            variable(:name => "nodenames_haproxy", :value => "#{list_of_running_instances.map{|a| "#{a.send :name}" }.join("\t")}")
-            variable(:name => "node_ips_haproxy",  :value => "#{list_of_running_instances.map{|a| "#{a.send :ip}" }.join("\t")}")
-          else
-            variable(:name => "nodenames_haproxy", :value => "generate('/usr/bin/env', '/var/lib/gems/1.8/bin/server-list-active', '-c', 'name')")
-            variable(:name => "node_ips_haproxy",  :value => "generate('/usr/bin/env', '/var/lib/gems/1.8/bin/server-list-active', '-c', 'ip')")
-          end
+          has_service(:name => "haproxy", :ensures => "running")
           
-          variable(:name => "ports_haproxy", :value => ([(self.respond_to?(:port) ? port : Base.port)].flatten))        
-          variable(:name => "forwarding_port", :value => (respond_to?(:forwarding_port) ? forwarding_port : Base.forwarding_port))
-          variable(:name => "proxy_mode", :value => (respond_to?(:proxy_mode) ? proxy_mode : Base.proxy_mode))
+          # has_exec(:name => "reloadhaproxy", :command => "/etc/init.d/haproxy reload", :requires => get_package("haproxy"))
 
           # These can also be passed in via hash
           has_remotefile(:name => "/etc/haproxy.cfg") do
             mode 644
-            # onlyif '$hostname == "master"'
-            requires 'Package["haproxy"]'
-            notify 'Service["haproxy"]'
+            requires get_package("haproxy")
+            notify get_service("haproxy")
             template File.join(File.dirname(__FILE__), "..", "templates/haproxy.conf")
           end
         end