staugaard-cloudmaster 0.1.3 → 0.1.4

data/app/policy_manual.rb

@@ -0,0 +1,36 @@
+require 'policy'
+
+module Cloudmaster
+
+  # Provide manual policy.
+  # This policy only changes the instances when requested to do so.
+  # This implementation uses a queue to convey manual requests
+  # to the policy module.
+  class PolicyManual < Policy
+    def initialize(reporter, config, instances)
+      super(reporter, config, instances)
+      @config = config
+      @sqs = AwsContext.instance.sqs
+      manual_queue_name = @config.append_env(config[:manual_queue_name])
+      @manual_queue = NamedQueue.new(manual_queue_name)
+    end
+
+    # Adjust never changes instances.
+    def adjust
+      n = 0
+      # Read all the messages out of the manual queue.
+      # Sum up all adjustments.
+      while true
+        messages = @manual_queue.read_messages(10)
+	break(n) if messages.size == 0
+        messages.each do |message|
+          msg = YAML.load(message[:body])
+          n += msg[:adjust]
+	  @manual_queue.delete_message(message[:receipt_handle])
+        end
+      end
+      # the value of the while is n
+    end
+  end
+
+end

data/app/policy_resource.rb

@@ -0,0 +1,110 @@
+
+require 'policy'
+
+module Cloudmaster
+
+  # Provides resource policy implementation.
+  # Instances managed under a resource policy are expected to issue
+  # periodic status messages, giving their estimated load (generally
+  # between 0 and 1).
+  class PolicyResource < Policy
+    # Each policy object gets the configuration and the instance collection.
+    def initialize(reporter, config, instances)
+      super(reporter, config, instances)
+      @config = config
+    end
+
+    # Activate the given number of shut_down instances.
+    # We prefer those with highest load.
+    # Return the number actually activated.
+    def activate_shut_down_instances(number_to_activate)
+      shutdown_instances = @instances.shut_down_instances.sort do |a,b|
+        b.load_estimate - a.load_estimate
+      end
+      shutdown_instances = shutdown_instances[0..number_to_activate]
+      shutdown_instances.each { |i| i.activate }
+      shutdown_instances.each { |i| @reporter.info("Activating instance ", i.id) }
+      shutdown_instances.size
+    end
+
+    # Shut down the given instances, by changing their state to shut_down.
+    def shut_down_instances(instances_to_shut_down)
+      instances = @instances.shut_down(instances_to_shut_down)
+      instances.each {|i| @reporter.info("Shutting down instance ", i.id) }
+      instances.size
+    end
+
+    # Shut down the given number of instances.
+    # Shut down the ones with the lowest load.
+    def shut_down_n_instances(number_to_shut_down)
+      return if number_to_shut_down <= 0
+      instances_with_lowest_load = @instances.sorted_by_lowest_load
+      instances_to_shut_down = instances_with_lowest_load.find_all do |instance|
+        # Don't stop instances before minimum_active_time
+        instance.minimum_active_time_elapsed?
+      end
+      shut_down_instances(instances_to_shut_down[0...number_to_shut_down])
+    end
+
+    # Stop any shut down instances with load below threshold.
+    # Also stop instances that have exceeded shut_down_interval.
+    def clean_up_shut_down_instances
+      idle_instances = @instances.shut_down_idle_instances
+      timeout_instances = @instances.shut_down_timeout_instances
+      stop_instances(idle_instances | timeout_instances)
+    end
+
+    # Adjust the instance pool up or down.
+    # If no instance are running, and there are requests in the work queue, start
+    # some.
+    # Additional instances are added if the load is too high.
+    # Instances are shut down, and then stopped if the load is low.
+    def adjust
+      depth = @config[:work_queue].empty_queue
+      if @instances.active_instances.size == 0
+        # capacity consumed by new arrivals
+        new_load = depth.to_f / @config[:queue_load_factor].to_f
+        initial = (new_load / @config[:target_upper_load].to_f).ceil
+	@reporter.info("Resource policy need initial #{initial}  depth: #{depth} new_load #{new_load}") if initial > 0
+        return initial
+      end
+      if depth > 0
+	@reporter.info("Resource policy residual depth: #{depth}")
+	return 0
+      end
+      # the total capacity remaining below the upper bound
+      excess_capacity = @instances.excess_capacity
+      if excess_capacity == 0
+        # need this many more running at upper bound
+	over_capacity = @instances.over_capacity
+        additional = (over_capacity / @config[:target_upper_load].to_f).ceil
+	@reporter.info("Resource policy need additional #{additional}  depth: #{depth} over_capacity #{over_capacity}")
+        return additional
+      end
+      # how many are needed to carry the total load at the lower bound
+      needed = (@instances.total_load / @config[:target_lower_load].to_f).ceil
+      if needed < @instances.size
+        excess = @instances.size - needed
+	@reporter.info("Resource policy need fewer #{excess}  depth: #{depth} needed #{needed}")
+	return -excess
+      end
+      return 0
+    end
+
+    # We are not using the default apply, because we want to:
+    #  * activate shut down instances, if posible, otherwise start
+    #  * shut down instances if fewer are needed
+    #  * stop inactive or expired shut_down instances
+    def apply
+      n = @limit_policy.adjust(adjust)
+      case 
+      when n > 0
+        n -= activate_shut_down_instances(n)
+        start_instances(n)
+      when n < 0
+        shut_down_n_instances(-n)
+      end
+      clean_up_shut_down_instances
+    end
+  end
+end

data/app/pool_configuration.rb

@@ -0,0 +1,172 @@
+require 'named_queue'
+require 'ec2_image_enumerator'
+
+module Cloudmaster
+
+#  All configuration parameters passed in through the constructor.
+#  Items with * must be defined
+#
+#  ==aws_config==
+#    aws_env -- used to form queue, instance, and s3 key names -- 
+#      typically development|test|production
+#    *aws_access_key -- the AWS access key
+#    *aws_secret_key -- the AWS secret key
+#    *aws_user -- the user name, used to build the image name
+#    *aws_bucket -- the bucket to use when storing the active set
+#    *aws_keypair -- full path name of the keypair file to use for 
+#      connecting to instances
+#
+#  ==config==
+#    ===GENERAL===
+#    *name -- the name of this config
+#    *policy -- :none, :job, :resource
+#    ===QUEUES===
+#    poll_interval -- how often to check work queue, etc (seconds)
+#    receive_count -- how many status messages to receive at once
+#    *work_queue -- name of work queue (aws_env)
+#    *status_queue -- name of status queue (aws_env)
+#    ===ACTIVE SET===
+#    active_set_type -- which active set algorithm to use: :none, :s3, :queue
+#    active_set_bucket -- the S3 bucket to use to store the active set
+#    active_set_key -- the S3 key used to store the active set (aws_env)
+#    active_set_interval -- how often to write active_set
+#    ===INSTANCE CREATION PARAMETERS===
+#    *ami_name -- the ami name to start and monitor (aws_env)
+#    key_pair_name -- the name if the keypair to start the instance with
+#    security_groups -- array of security group names to start the instance with
+#    instance_type -- the smi instance type to create
+#    user_data -- instance data made available to running instance 
+#        through http://169.254.169.254/latest/user-data
+#        This is given as a hash, which is serialized by cloudmaster.
+#     
+#    ===INSTANCE MANAGEMENT POLICIES===
+#    policy_interval -- how often to apply job or resource policy
+#    audit_instance_interval -- how often (in minutes) to audit instances (-1 for never)
+#    maximum_number_of_instances -- the max number to allow
+#    minimum_number_of_instances -- the min number to allow
+#    ===INSTANCE START POLICIES===
+#    start_limit -- how many instances to start at one time
+#    ===INSTANCE STOP POLICIES===
+#    stop_limit -- how many to stop at one time
+#    minimum_lifetime -- don't stop an instance unless it has run this long (minutes)
+#    minimum_active_time -- the minimum amount of time (in minutes) that an instance
+#      may remain in the active state
+#    watchdog_interval -- if a machine does not report status in this interval, it is
+#      considered to be hung, and is stopped
+#    ===JOB POLICIES===
+#    start_threshold -- if work queue size is greater than start_threshold * number of
+#      active instances, start more instances
+#    idle_threshold -- if more than idle_threshold active instances with load 0 
+#      exist, stop some of them
+#    ===RESOURCE POLICIES===
+#    target_upper_load -- try to keep instances below this load
+#    target_lower_load -- try to keep instances above this load
+#    queue_load_factor -- the portion of the load that a single queue entry represents.
+#      If a server can serve a maximum of 10 clients, then this is 10.
+#    shut_down_threshold -- stop instances that have load_estimate below this value
+#    shut_down_interval -- stop instances that have been in shut_down state for
+#      longer than this interval
+#    ===MANUAL POLICIES===
+#    manual_queue_name -- the name of the queue used to send manual instance adjustments
+#    ===REPORTING===
+#    summary_interval -- how often to give summary 
+#    instance_log -- if set, it is a patname to a directory where individual log files
+#      are written for each instance
+#    instance_report_interval -- how often to show instance reports
+
+  # PoolConfiguration holds the configuration parameters for one pool.
+  # It also stores aws parameters and defaults, providing a single lookup mechanism
+  # for all.
+  # If lookup files, then it raise an exception.
+
+  class PoolConfiguration
+    # Create a new PoolConfiguration.  The default parameters
+    # are used if the desired parameter is not given.
+    def initialize(aws_config, default, config)
+      # these parameters merge the defaults and the given parbameters
+      # merged parameters are also evaluated
+      @merge_params = [:user_data]
+      @aws_config = aws_config
+      @default = default
+      @config = config
+    end
+    
+    # Get a parameter, either from aws_config, config or default.
+    # Don't raise an exception if there is no value.
+    def get(param)
+      @aws_config[param] || @config[param] || @default[param]
+    end
+
+    # Get a parameter, either from config or from default.
+    # Raise an exception if there is none.
+    def [](param)
+      if @default.nil?
+        raise "Missing defaults"
+      end
+      config_param = @aws_config[param] || @config[param]
+      if (res = config_param || @default[param]).nil?
+        raise "Missing config: #{param}"
+      end
+      begin
+        if @merge_params.include?(param)
+	 # fix up default param if needed -- it must be a hash
+	  @default[param] = {} if @default[param].nil? 
+          @default[param] = eval(@default[param]) if @default[param].is_a?(String)
+          if config_param 
+            @default[param].merge(eval(config_param))
+	  else
+            @default[param]
+	  end
+        else
+          res
+        end
+      rescue
+        raise "Config bad format: #{param} #{config_param} #{$!}"
+      end
+    end
+
+    # Store (create or replace) a parameter.
+    def []=(param, val)
+      @config[param] = val
+    end
+
+    def append_env(name)
+      aws_env = @aws_config[:aws_env]
+      aws_env.nil? || aws_env == '' ? name : "#{name}-#{aws_env}"
+    end
+
+    # Test to see that the derived parameters are valid.
+    def valid?
+      @config[:ami_id] && 
+        @config[:work_queue] && @config[:work_queue].valid? && 
+        @config[:status_queue] && @config[:status_queue].valid?
+    end
+
+    # Looks up a queue given its name.
+    # Stores the result in config under the given key (if given).
+    # Returns the queue.
+    # Raises an exception if none found.
+    def setup_queue(key, name)
+      return nil unless name
+      name = append_env(@config[name])
+      queue = NamedQueue.new(name)
+      raise "Bad configuration -- no queue #{name}" if !queue
+      @config[key] = queue if key
+      queue
+    end
+
+    # Looks up the image, given its name.
+    # Stores the result in config under the given key (if given).
+    # Returns the image.
+    # Raises an exception if none found.
+    def setup_image(key, name)
+      return nil unless name
+      name = append_env(@config[name])
+      image = EC2ImageEnumerator.new.find_image_id_by_name(name)
+      raise "Bad configuration -- no image #{name}" if !image
+      @config[key] = image if key
+      image
+    end
+
+  end
+end

data/app/pool_manager.rb

@@ -0,0 +1,239 @@
+require 'periodic'
+require 'pp'
+require 'logger'
+require 'reporter'
+require 'instance_pool'
+require 'aws_context'
+require 'policy_factory'
+require 'active_set_factory'
+require 'status_parser_factory'
+require 'logger_factory'
+require 'policy'
+
+module Cloudmaster
+
+#  PoolManager
+#
+#  Manages one InstancePool, which is collections of EC2 instances 
+#  running the same image.  
+#  The InstancePoolMaanger is responsible for starting and terminating 
+#  instances.
+#  It's policies are meant to balance acceptable performance while 
+#  minimizing cost.
+#  To help achieve this goal, the PoolManager receives 
+#  status reports from instances, through a status queue.
+#
+#  Two classes of policies are defined: job and resource. 
+#  These roughly correspond to stateless and stateful services.
+#
+#  ==Job Policy==
+#  In the job policy, instances are assigned work through a work queue.
+#  * Each request is stateless, and can be serviceed by any instance.
+#  * Each instance processes one request at a time.
+#  * Each instance is either starting_up or active.  
+#  * Once it is active, it is either busy (load 1.0) or idle (load 0.0).
+#  At startup, the instance reports when it is ready to begin processing, and
+#  enters the active state.
+#  Each instance reports the load through the status queue when it 
+#  starts/stops processing a job.
+#
+#  The job policy aims to keep the work queue to a reasonable size while not
+#  maintaining an excessive number of idle instances.
+#
+#  ==Resource Policy==
+#  Instance managed by theresource policy have stateful associations with 
+#  clients, and provide them services on demand.  
+#  * Each instance processes requests made by clients as requested.
+#  * An external entity (the alllocator) assigns clients to instances 
+#    based on an instance report, which  lists the active instances 
+#    and their associated load.  
+#  * The instance report (called the active set) is stored in
+#    S3, at a configurable bucket and key.
+#  * The allocator assigns clients to instances, and also creates a 
+#    work-queue entry each time it assigns a new client.
+#  * The allocator is expected to assign clients only to those instances 
+#    listed in the active set.
+#  *  The work queue is emptied by cloudmaster.
+#  *  Each instance may be starting_up, active, or shutting_down.
+#  *  At startup, the instance reports when it is ready to begin processing, 
+#     and enters the active state.
+#  *  The policy decides when to shut down an instance.  
+#     It puts it in the shut_down state, but does not stop
+#     it immediately (to avoid disturbing existing clients).
+#     Instances in shutting_down state with zero load, or who have
+#     remained in this state for an excessive time are stopped.
+#  *  Active instances are available to accept new clients; 
+#     shutting_down instances are not.
+#  During any given time period, each instance can be partially busy (load 
+#  between 0.0 and 1.0)
+#  Each instance periodically reports is load estimate for that period through 
+#  the status queue.
+#  The resource policy seeks to maintain a load between an 
+#  upper threshold and a lower threshold.  
+#  It starts instances or stops them to achieve this.
+
+  class PoolManager
+    attr_reader :instances, :logger     # for testing only
+
+    # Set up PoolManager.
+    # Creates objects used to access SQS and EC2.
+    # Creates instance pool, policy classes, repoter, and queues.
+    # Actual processing does not start until "run" is called.
+    def initialize(config)
+      # set up AWS access objects
+      keys = [ config[:aws_access_key], config[:aws_secret_key]]
+      aws = AwsContext.instance
+      @ec2 = aws.ec2(*keys)
+      @sqs = aws.sqs(*keys)
+      @s3 = aws.s3(*keys)
+      @config = config
+
+      # set up reporter
+      @logger = LoggerFactory.create(@config[:logger], @config[:logfile])
+      @reporter = Reporter.setup(@config[:name], @logger)
+
+      # Create instance pool.
+      # Used to keep track of instances in the pool.
+      @instances = InstancePool.new(@reporter, @config)
+
+      # Create a policy class
+      @policy = PolicyFactory.create(@config[:policy], @reporter, @config, @instances)
+
+      # Create ActiveSet
+      @active_set = ActiveSetFactory.create(@config[:active_set_type], @config)
+
+      # Create StatusParser
+      @status_parser = StatusParserFactory.create(@config[:status_parser])
+
+      unless @config[:instance_log].empty?
+        @reporter.log_instances(@config[:instance_log])
+      end
+
+      # Look up the work queues and the image from their names.
+      # Have policy do most of the work.
+      @work_queue = @config.setup_queue(:work_queue, :work_queue_name)
+      @status_queue = @config.setup_queue(:status_queue, :status_queue_name)
+      @ami_id = @config.setup_image(:ami_id, :ami_name)
+
+      @keep_running = true
+    end
+    
+    # Main loop of cloudmaster
+    # 
+    # * Reads and processes status messages.
+    # * Starts and stops instances according to policies
+    # * Detects hung instances, and stops them.
+    # * Displays periodic reports.
+    def run(end_time = nil)
+      summary_period = Periodic.new(@config[:summary_interval].to_i)
+      instance_report_period = Periodic.new(@config[:instance_report_interval].to_i)
+      policy_period = Periodic.new(@config[:policy_interval].to_i)
+      active_set_period = Periodic.new(@config[:active_set_interval].to_i * 60)
+      audit_instances_period = Periodic.new(@config[:audit_instance_interval].to_i * 60)
+
+      # loop reading messages from the status queue
+      while keep_running(end_time) do
+        # upate instance list and get queue depth
+	audit_instances_period.check do
+          @instances.audit_existing_instances
+	end
+
+        @work_queue.read_queue_depth
+        break unless @keep_running
+
+        # start first instance, if necessary, and ensure the
+        #  number of running instances stays between maximum and minimum
+        @policy.ensure_limits
+        break unless @keep_running
+
+        # handle status and log messages
+        process_messages(@config[:receive_count].to_i)
+		    
+        # update public dns (for new instances) and show summary reports
+        @instances.update_public_dns_all
+        summary_period.check do
+          @reporter.info("Instances: #{@instances.size} Queue Depth: #{@work_queue.queue_depth}")
+        end
+        instance_report_period.check do
+          @reporter.info("---Instance Summary---")
+          @instances.each do |instance|
+            @reporter.info("  #{instance.id} #{instance.report}\n")
+          end
+          @reporter.info("----------------------")
+        end
+        break unless @keep_running
+        
+        # Based on queue depth and load_estimate, make a decision on
+        # whether to start or stop servers.
+        policy_period.check { @policy.apply }
+
+        active_set_period.check { update_active_set }
+
+        # Stop instances that have not given recent status.
+        @policy.stop_hung_instances
+        break unless @keep_running
+
+        Clock.sleep @config[:poll_interval].to_i
+      end
+    end
+    
+    # Shut down the manager.
+    # This may take a little time.
+    def shutdown
+      @keep_running = false
+    end
+
+  private
+
+    # Process a batch of status and log messaage.
+    # Status messages update the instance usage information, and
+    # log messages are just logged.
+    # Observed behavior is that only one message is returned per call
+    # to SQS, no matter how many are requested.
+    def process_message_batch(count)
+      # read some messages
+      messages = @status_queue.read_messages(count)
+      messages.each do |message|
+        # parse message 
+	msg = @status_parser.parse_message(message[:body])
+        case msg[:type]
+        when "status"
+          # save the status and load_estimate
+          @instances.update_status(msg)
+        when "log"
+          # just log the message
+          @reporter.info(msg[:message], msg[:instance_id])
+        end
+        # delete the message once it has been processed
+        @status_queue.delete_message(message[:receipt_handle])
+      end
+      messages.size
+    end
+    
+    # Process messages (up to count)
+    # Continue until there are no messages remaining.
+    def process_messages(count)
+      n_remaining = count
+      while n_remaining > 0
+        n = process_message_batch(n_remaining)
+        break if n == 0
+        n_remaining -= n
+      end
+    end
+    
+    # Write active set if it has changed since the last write.
+    def update_active_set
+       @active_set.update(@instances.active_set)
+    end
+
+    # Returns true if the manager should keep running.
+    def keep_running(end_time)
+      if end_time && Clock.now > end_time
+        false
+      else
+        @keep_running
+      end
+    end
+
+  end
+end

data/app/pool_runner.rb

@@ -0,0 +1,54 @@
+require 'pool_configuration'
+require 'pool_manager'
+
+module Cloudmaster
+
+  #  oolRunner
+  #
+  #  Manages separate PoolManagers, each in a separate thread.
+  #  
+  #  Knows how to start (run) and stop (shutdown) the pools.
+  #
+  # Creates a thread for each pool in config, and runs a PoolManager in it.
+  # This needs to be passed a configuration, normally a InifileConfig object
+  # The configuration object contains all the information needed to control
+  # the pools, including the number of pools and each one's characteristics.
+  class PoolRunner
+    attr_reader :pool_managers   # for testing only
+    # Create empty runner.  Until the run method is called, the
+    #  individual pool managers are not created.
+    def initialize(config)
+      @config = config
+      @pool_managers = []
+      Signal.trap("INT") do
+        self.shutdown
+      end
+    end
+
+    # Create each of the pool managers described in the configuration.
+    # We can limit the amount of time it runs, for testing purposes only
+    # In testing we can call run again after it returns, so we make sure
+    # that we only create pool managers the first time through.
+    def run(limit = nil)
+     if @pool_managers == []
+        @config.pools.each do |pool_config|
+          # Wrap pool config parameters up with defaults.
+          config = PoolConfiguration.new(@config.aws, @config.default, pool_config)
+          @pool_managers << PoolManager.new(config)
+        end
+      end
+      threads = []
+      @pool_managers.each do |pool_manager|
+        threads << Thread.new(pool_manager) do |mgr|
+          mgr.run(limit)
+        end
+      end
+      threads.each { |thread| thread.join }
+    end
+
+    # Shut down each of the pool managers.
+    def shutdown
+      @pool_managers.each { |mgr| mgr.shutdown }
+    end
+  end
+end

data/app/reporter.rb

@@ -0,0 +1,81 @@
+require 'instance_logger'
+
+module Cloudmaster
+
+  # Creates and outputs log messages
+  # These are formatted with a timestamp and an instance name.
+  # This remembers the log device, which is anything with puts.
+  # This is treated as a global.  It is initialized by calling "Reporter.setup"
+  # and then anyone can get a copy by calling "Reporter.instance".  
+  class Reporter
+    attr_accessor :level
+
+    NONE = 0
+    ERROR = 1
+    WARNING = 2
+    INFO = 3
+    TRACE = 4
+    DEBUG = 5
+    ALL = 10
+
+    # Reporter displays the given name on every line.
+    # reports go to the given log (an IO).
+    def initialize(name, log)
+      @level = ALL
+      @name = name
+      @log = log || STDOUT
+      @instance_logger = nil
+    end
+
+    def Reporter.setup(name, log)
+      new(name, log)
+    end
+
+    def log_instances(dir)
+      @instance_logger = InstanceLogger.new(dir)
+    end
+
+    # Log a message 
+    def log(message, *opts)
+      send_to_log("INFO:", message, *opts)
+    end
+
+    def err(msg, *opts)
+      send_to_log("ERROR:", msg, *opts) if @level >= ERROR
+    end
+    alias error err
+
+    def warning(msg, *opts)
+      send_to_log("WARNING:", msg, *opts) if @level >= WARNING
+    end
+
+    def info(msg, *opts)
+      send_to_log("INFO:", msg, *opts) if @level >= INFO
+    end
+
+    def trace(msg, *opts)
+      send_to_log("TRACE:", msg, *opts) if @level >= TRACE
+    end
+
+    def debug(msg, *opts)
+      send_to_log("DEBUG:", msg, *opts) if @level >= DEBUG
+    end
+
+    private
+
+    def send_to_log(type, message, instance_id = nil)
+      msg = [type, format_timestamp(Clock.now), @name]
+      msg << instance_id if instance_id
+      msg << message
+      message = msg.join(' ')
+      @log.puts(message)
+      if instance_id && @instance_logger
+	@instance_logger.puts(instance_id, message)
+      end
+    end
+
+    def format_timestamp(ts)
+       "#{Clock.now.strftime("%m-%d-%y %H:%M:%S")}"   
+    end
+  end
+end