auser-poolparty 1.0.0 → 1.1.1

data/lib/poolparty/monitors/stats_monitor.rb

@@ -0,0 +1,183 @@
+require ::File.dirname(__FILE__)+"/monitor_rack.rb"
+
+module Monitors
+    
+  class Stats 
+    attr_reader :stats, :request
+    attr_accessor :response
+    
+    def initialize(env, o={})
+      @env = env
+      @request = Rack::Request.new env
+      @response = Rack::Response.new
+      
+      begin
+        @cloud = JSON.parse( open('/etc/poolparty/clouds.json' ).read )
+      rescue 
+        @cloud = ::PoolParty::Default.options.merge({"options" => 
+          {"rules" => {"expand" => PoolParty::Default.expand_when, 
+                        "contract" => PoolParty::Default.contract_when
+                      }
+          }
+        })
+      end      
+      # Our cloud.options.rules looks like
+      #  {"expand_when" => "load > 0.9", "contract_when" => "load < 0.4"}
+      # We set these as rules on ourselves so we can use aska to parse the rules
+      # So later, we can call vote_rules on ourself and we'll get back Aska::Rule(s)
+      # which we'll call valid_rule? for each Rule and return the result
+      @cloud["options"]["rules"].each do |name, rul|
+        r = Aska::Rule.new(rul)
+        rule(name) << r
+      end
+      log << "#{Time.now.strftime("%Y-%m-%d-%H-%M")}, #{stats.to_json}\n"
+    end
+    
+    def default
+      stats.to_json
+    end
+
+    def log(log_file_path="/var/log/poolparty/stats_monitor.log")
+      ::File.file? log_file_path
+      @logfile ||= ::File.new(log_file_path, 'a')
+    end
+
+    def default
+      begin
+        if !request.params || request.params.empty?
+          default_stats
+        else
+          stats[request.params[0].to_sym] ||= self.send(request.params[0])
+          stats[request.params[0].to_sym]
+          stats.to_json
+        end
+      rescue Exception => e
+        response.fail!
+        "Error: #{e}"
+      end
+    end
+    alias :get :default
+
+    def put
+      if d = JSON.parse(request.params)
+        hsh = d.reject {|ip, _node| ip == my_ip }
+        stats.merge!(hsh)
+        handle_election
+      else
+        "boom"
+      end
+    end
+    alias :update :put
+
+    # Handle the elections
+    def handle_election
+      # Ballots look like:
+      # host => ["contract"]
+      candidates = {:expand => 0, :contract => 0}
+      candidates.each do |action, ballots|
+        stats.each do |ip, node_hsh|
+          candidates[action]+=1 if node_hsh["nominations"] && node_hsh["nominations"].include?(action.to_s)
+        end
+      end
+      # TODO: Move?
+      # Expand the cloud if 50+% of the votes are for expansion
+      # Contract the cloud if 51+% of the votes are for contraction
+      if (candidates[:expand] - candidates[:contract])/stats.keys.size > 0.5
+        %x[/usr/bin/server-cloud-elections expand] unless elected_action == "expand"
+        @elected_action = "expand"
+      elsif (candidates[:contract] - candidates[:expand])/stats.keys.size > 0.5
+        %x[/usr/bin/server-cloud-elections contract] unless elected_action == "contract"
+        @elected_action = "contract"
+      end      
+
+      reload_data!
+      stats[my_ip]["elected_action"] = @elected_action if @elected_action
+      log << "#{Time.now.strftime("%Y-%m-%d-%H-%M")}, #{stats.to_json}\n"
+      stats.to_json
+    end
+
+
+    def elected_action
+      @elected_action ||= nil
+    end
+
+    def rules
+      @rules ||= {}
+    end
+
+    def rule(name)
+      rules[name] ||= []
+    end
+
+    def default_stats
+      %w(load nominations).each do |var|
+        stats[my_ip][var] ||= self.send(var.to_sym)
+      end
+      stats
+    end
+
+    def stats
+      @stats ||= {my_ip  => {}}
+    end
+
+    def load
+      %x{"uptime"}.split[-3].to_f
+    end
+
+    def instances
+      # res = PoolParty::Neighborhoods.load_default.instances
+      res ||= %x[/usr/bin/server-list-active internal_ip].split("\t")
+      res
+    end
+
+    def can_expand?
+      instances.size < max_instances
+    end
+
+    def can_contract?
+      instances.size > min_instances
+    end
+
+    def min_instances
+      (@cloud["options"]["minimum_instances"] || PoolParty::Default.minimum_instances).to_i
+    end
+
+    def max_instances
+      (@cloud["options"]["maximum_instances"] || PoolParty::Default.maximum_instances).to_i
+    end
+
+    def nominations
+      load = stats[my_ip]["load"] ||= self.send(:load)
+      stats[my_ip]["nominations"] ||= rules.collect do |k,cld_rules|
+        t = cld_rules.collect do |r|
+          # If the comparison works
+          if self.send(r.key.to_sym).to_f.send(r.comparison, r.var.to_f)
+            # if we are facing an expansion rule
+            if k =~ /expand/
+              k if can_expand?
+            # if we are facing a contraction rule
+            elsif k =~ /contract/
+              k if can_contract?
+            else
+              k
+            end
+          end
+        end.compact
+      end.flatten.compact
+    end
+
+    def my_ip
+      @my_ip ||= ohai["ipaddress"]
+    end
+
+    def ohai
+      @ohai ||= JSON.parse(%x[ohai])
+    end
+
+    def reload_data!
+      @stats[my_ip] = {}
+      instances.each {|inst| @stats[inst] = {} }
+    end    
+    
+  end
+end

data/lib/poolparty/monitors.rb

@@ -0,0 +1,6 @@
+require "rubygems"
+require "json"
+require ::File.join(::File.dirname(__FILE__), 'monitors/', 'monitor_rack.rb') 
+
+module PoolParty
+end

data/lib/poolparty/net/init.rb

@@ -0,0 +1,11 @@
+# Load the core net libraries.  These are neccessary for any of the remoter_bases to function.
+%w(remote_instance messenger remote_bases remoter_base).each do |file|
+  require File.join(::File.dirname(__FILE__),file+'.rb')
+end
+
+# Register available remoter_bases
+Dir["#{::File.dirname(__FILE__)}/remoter_bases/*/*.rb"].each do |base| 
+  name = File.join(::File.basename(base, ::File.extname(base)))
+  require base
+  register_remote_base name
+end

data/lib/poolparty/net/messenger.rb

@@ -9,7 +9,7 @@ module PoolParty
     def with_socket(testing=false, &block)
       host = testing ? "localhost" : (master.ip)
       vputs "Pinging #{host} with the messenger"
-      socket = TCPSocket.open(host, Base.messenger_client_port)
+      socket = TCPSocket.open(host, Default.messenger_client_port)
       out = yield(socket)
       socket.close
       out
@@ -52,21 +52,4 @@ module PoolParty
     end
     
   end
-end
-
-module PoolParty
-  module Cloud
-    class Cloud
-      include PoolParty::Messenger
-      
-      def get_current_nodes
-        nodes = messenger_send!("get_current_nodes")
-        nodes.split(" ").map {|a| a.split(/@/)[-1] }
-      end
-      
-      def reconfigure_cloud!(msg="force_reconfig")
-        messenger_cast!(msg)
-      end
-    end
-  end
 end

data/lib/poolparty/net/remote_bases.rb

@@ -0,0 +1,21 @@
+require File.dirname(__FILE__) + "/remoter_base"
+
+class Object
+  def remote_bases
+    $remote_bases ||= []
+  end
+  # Register the remoter base in the remote_bases global store
+  def register_remote_base(*args)
+    args.each do |arg|
+      base_name = "#{arg}".downcase.to_sym
+      (remote_bases << base_name) unless remote_bases.include?(base_name)
+    end
+  end
+  alias :available_bases :remote_bases
+end
+
+Dir["#{File.dirname(__FILE__)}/remote_bases/*.rb"].each do |base| 
+  name = ::File.basename(base, ::File.extname(base))
+  require base
+  register_remote_base name
+end

data/lib/poolparty/net/remote_instance.rb

@@ -1,19 +1,14 @@
-require File.dirname(__FILE__) + "/remoter"
-
 module PoolParty  
   module Remote
     
     class RemoteInstance
-      include Remote
-      include Configurable
-      include CloudResourcer
+      include Dslify
       
-      def initialize(opts, parent=self)        
-        run_setup(parent)
+      def initialize(opts={}, containing_cloud=nil)
+        @parent = containing_cloud
 
-        set_vars_from_options(parent.options) if parent && parent.respond_to?(:options)
-        set_vars_from_options(opts) unless opts.nil? || opts.empty?
-        
+        set_vars_from_options(containing_cloud.options) if containing_cloud && containing_cloud.respond_to?(:options)
+        set_vars_from_options(opts) if opts.is_a?(Hash)
         on_init
       end
       
@@ -26,50 +21,69 @@ module PoolParty
       end
       
       # Is this remote instance the master?
+      # DEPRECATE
       def master?
         name == "master"
       end
       
       # The remote instances is only valid if there is an ip and a name
       def valid?
-        !(ip.nil? || name.nil?)
+        (ip.nil? || name.nil?) ? false : true
       end
       
       # Determine if the RemoteInstance is responding
       def responding?
-        !responding.nil?
+        running?
+        # !responding.nil? #TODO MF this needs to actually ping the node or something similar.  stubbed to running? for now
       end
       
       # This is how we get the current load of the instance
       # The approach of this may change entirely, but the usage of
       # it will always be the same
       def load
-        current_load ||= 0.0
+        current_load ||= 0.0  #NOTE MF: returning 0.0 seems like a bad idea here.  should return nil if we dont have a real value
       end
-            
+      
+      # Note, the next 4 methods will be overridden by the cloud specific remoter_base
       # Is this instance running?
       def running?
-        !(status =~ /running/).nil?
+        true
       end
       # Is this instance pending?
       def pending?
-        !(status =~ /pending/).nil?
+        false
       end
       # Is this instance terminating?
       def terminating?
-        !(status =~ /shutting/).nil?
+        false
       end
       # Has this instance been terminated?
       def terminated?
-        !(status =~ /terminated/).nil?
+        false
       end
       
       # Printing. This is how we extract the instances into the listing on the 
       # local side into the local listing file
       def to_s
-        "#{name}\t#{ip}\t#{instance_id}"
+        "#{name}\t#{ip}\t#{instance_id rescue ""}"
+      end
+      
+      # Class method to disect a neighborhood line
+      def self.hash_from_s(s)
+        arr = s.split("\t")
+        {:name => arr[0], :ip => arr[1]}
+      end
+      
+      def self.to_s(hsh)
+        new(hsh).to_s
+      end
+      
+      #TODO: Diet the next 5 commands
+      def dependency_resolver_command
+        cloud.dependency_resolver_command
       end
       
+      #FIXME: deprecate puppet specific commands in this class
       def puppet_runner_command
         self.class.send :puppet_runner_command
       end
@@ -83,13 +97,8 @@ module PoolParty
       def self.puppet_rerun_commad
         puppet_runner_command
       end
-      def my_cloud
-        @pa = parent
-        while !(@pa.is_a?(PoolParty::Cloud::Cloud) || @pa.nil? || @pa == self)
-          @pa = @pa.parent
-        end
-        @pa
-      end
+      #
+      
       def hosts_file_listing_for(cl)
         string = (cl.name == cloud.name) ? "#{name}.#{my_cloud.name}\t#{name}" : "#{name}.#{my_cloud.name}"
         "#{internal_ip}\t#{string}"
@@ -97,4 +106,4 @@ module PoolParty
     end
     
   end  
-end
+end

data/lib/poolparty/net/remoter/cloud_control.rb

@@ -0,0 +1,251 @@
+require "ping"
+
+module PoolParty
+  module Remote
+    
+    # TODO: Rename and modularize the @inst.status =~ /pending/ so that it works on all 
+    # remoter_bases
+    def launch_instance!(o={}, &block)
+      @inst = launch_new_instance!( o )
+      wait "2.seconds"
+      500.times do |i|
+        if @inst.status =~ /pending/
+          sleep(2)
+          @inst = describe_instance(@inst)          
+        end
+      end
+      when_instance_is_responding @inst do
+        block.call(@inst) if block
+        after_launch_instance(@inst)
+      end
+      @inst
+    end
+    
+    # Called after an instance is launched
+    def after_launch_instance(instance=nil)      
+    end
+    
+    def when_instance_is_responding(inst, &block)
+      if ping_port(inst.ip, 22)
+        block.call if block
+      else
+        raise "Instance not responding at #{inst.ip}"
+      end
+    end
+    
+    # A convenience method for waiting until there are no more
+    # pending instances and then running the block
+    def when_no_pending_instances(&block)
+      reset!
+      if list_of_pending_instances && list_of_pending_instances.size == 0
+        vputs "" # Clear the terminal with a newline
+        block.call if block
+      else
+        vprint "."
+        wait "5.seconds"
+        when_no_pending_instances(&block)
+      end
+    end
+    
+    # Stub method for the time being to handle expansion of the cloud
+    def can_expand_cloud?(force=false)
+      (are_too_few_instances_running? || are_expansion_rules_valid? ) || force || false
+    end
+    def are_expansion_rules_valid?
+      valid_rules?(:expand_when)
+    end
+    # Stub method for the time being to handle the contraction of the cloud
+    def can_contract_cloud?(force=false)
+      return true if force
+      ((are_any_nodes_exceeding_minimum_runtime? and are_too_many_instances_running?) || are_contraction_rules_valid?) || false
+    end
+    def are_contraction_rules_valid?
+      valid_rules?(:contract_when)
+    end
+    # Expand the cloud
+    # If we can start a new instance and the load requires us to expand
+    # the cloud, then we should request_launch_new_instances
+    # Wait for the instance to boot up and when it does come back
+    # online, then provision it as a slave, this way, it is ready for action from the
+    # get go
+    def expand_cloud_if_necessary(force=false)
+      if can_start_a_new_instance? && can_expand_cloud?(force)
+        vputs "Expanding the cloud based on load"
+        @num = 1
+        @num.times do |i|
+          list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")          
+          reset!
+          vputs "request_launch_new_instances: #{@num}"
+          provision_slaves_from_n(@num)
+          after_launched
+        end
+      end
+    end
+    # Contract the cloud
+    # If we can shutdown an instnace and the load allows us to contract
+    # the cloud, then we should request_termination_of_non_master_instance
+    def contract_cloud_if_necessary(force=false)
+      if can_shutdown_an_instance? && can_contract_cloud?(force)
+        vputs "Shrinking the cloud by 1"
+        before_shutdown
+        request_termination_of_non_master_instance
+      end
+    end
+    
+    # List calculation methods
+    # 
+    # Are the minimum number of instances running?
+    def minimum_number_of_instances_are_running?
+      list_of_running_instances.size >= minimum_instances.to_i
+    end
+    # Are the minimum number of instances NOT running?
+    def minimum_number_of_instances_are_not_running?
+      !(minimum_number_of_instances_are_running?)
+    end
+    # Can we shutdown an instance?
+    def can_shutdown_an_instance?
+      list_of_running_instances.size > minimum_instances.to_i
+    end
+    # Are too few instances running?
+    def are_too_few_instances_running?
+      list_of_running_instances.size < minimum_instances.to_i
+    end
+    # Are there more instances than allowed?
+    def are_too_many_instances_running?
+      list_of_running_instances.size > maximum_instances.to_i
+    end
+    
+    ########
+    # TODO: deprecate methods below here (only if they are deprecate-able)
+    ########
+    
+    # Request to launch a number of instances
+    def request_launch_new_instances(num=1)
+      out = []
+      num.times {out << launch_new_instance!(options) }
+      out
+    end
+    def request_launch_master_instance
+      @inst = launch_new_instance!
+      wait "5.seconds"
+      when_no_pending_instances do
+        vputs "Master has launched"
+        reset!
+        after_launch_master(@inst)
+      end
+    end
+    
+    
+    def after_launch_master(inst=nil)
+      vputs "After launch master in remoter"
+    end
+    # Let's terminate an instance that is not the master instance
+    def request_termination_of_non_master_instance
+      inst = nonmaster_nonterminated_instances.last
+      terminate_instance!(inst.instance_id) if inst
+    end
+    # Can we start a new instance?
+    def can_start_a_new_instance?
+      maximum_number_of_instances_are_not_running? && list_of_pending_instances.size == 0
+    end
+    # Are the maximum number of instances not running?
+    def maximum_number_of_instances_are_not_running?
+      list_of_running_instances.size < maximum_instances.to_i
+    end
+    # Are the maximum number of instances running?
+    def maximum_number_of_instances_are_running?
+      list_of_running_instances.size >= maximum_instances.to_i
+    end
+    # Launch new instance while waiting for the number of pending instances
+    #  to be zero before actually launching. This ensures that we only
+    #  launch one instance at a time
+    def request_launch_one_instance_at_a_time
+      when_no_pending_instances { launch_new_instance! }
+    end
+
+    # A convenience method for waiting until all the instances have an ip
+    # assigned to them. This is useful when shifting the ip addresses
+    # around on the instances
+    def when_all_assigned_ips(&block)
+      reset!
+      if list_of_nonterminated_instances.select {|a| a.ip == 'not.assigned' }.empty?          
+        block.call if block
+      else
+        vprint "."
+        wait "5.seconds"
+        when_all_assigned_ips(&block)
+      end
+    end
+    def running_instance_ips
+      remote_instances_list.select {|inst| 
+        inst.running? and inst.ip!='not.assigned'
+      }.collect{|n| n.ip}
+    end
+    
+    # This will launch the minimum_instances if the minimum number of instances are not running
+    # If the minimum number of instances are not running and if we can start a new instance
+    def launch_minimum_number_of_instances        
+      if can_start_a_new_instance? && !minimum_number_of_instances_are_running?         
+        list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")
+        reset!
+        launch_minimum_number_of_instances
+        provision_slaves_from_n(minimum_instances.to_i)
+        after_launched
+      end
+    end
+    
+    def provision_slaves_from_n(num=1)
+      vputs "In provision_slaves_from_n: #{num}"
+      reset!
+      when_no_pending_instances do
+        vputs "Waiting for 10 seconds"
+        wait "10.seconds" # Give some time for ssh to startup          
+        @num_instances = list_of_running_instances.size
+        vputs "(@num_instances - (num))..(@num_instances): #{(@num_instances - (num))..(@num_instances)}"
+        last_instances = nonmaster_nonterminated_instances[(@num_instances - (num))..(@num_instances)]
+        last_instances.each do |inst|
+          vputs "Provision slave: #{inst}"
+          verbose ? provisioner_for(inst).install(testing) : hide_output { provisioner_for(inst).install(testing)}
+        end
+        # PoolParty::Provisioner.reconfigure_master(self)
+      end
+    end
+    # Launch the master and let the master handle the starting of the cloud
+    # We should only launch an instance if there are no pending instances, in the case 
+    # that the master has launched, but is still pending
+    # and if the master is not running AND we can start a new instance
+    # Then wait for the master to launch
+    def launch_and_configure_master!(testing=false)
+      vputs "Requesting to launch new instance"
+      dputs "Launching master"
+      request_launch_master_instance if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running? && !testing
+      reset!
+      unless testing
+        vputs ""
+        vputs "Waiting for there to be no pending instances..."
+        when_no_pending_instances do
+          when_all_assigned_ips {wait "20.seconds"}
+          vputs ""
+          vputs "Provisioning master..."
+          # cleanup_storage_directory
+          @provisioner = PoolParty::Provisioner::Capistrano.new(master, self, :ubuntu)
+          verbose ? @provisioner.install(testing) : hide_output { @provisioner.install(testing) }
+        
+          after_launched
+        end
+      end
+    end
+    def list_of_nodes_exceeding_minimum_runtime
+      list_of_running_instances.reject{|i| i.elapsed_runtime < minimum_runtime}
+    end
+    
+    def are_any_nodes_exceeding_minimum_runtime?
+      !list_of_nodes_exceeding_minimum_runtime.blank?
+    end
+    # Is there a node that is running with the name master
+    def is_master_running?
+      !list_of_running_instances.select {|a| a.name == "master"}.first.nil?
+    end    
+    
+  end
+end