staugaard-cloudmaster 0.1.1

data/lib/aws_context.rb

@@ -0,0 +1,165 @@
+
+require 'AWS/SQS'
+require 'AWS/EC2'
+require 'AWS/S3'
+require 'AWS/SimpleDB'
+require 'SafeAWS/SQS'
+require 'SafeAWS/EC2'
+require 'SafeAWS/S3'
+require 'SafeAWS/SimpleDB'
+require 'RetryAWS/SQS'
+require 'RetryAWS/EC2'
+require 'RetryAWS/S3'
+require 'RetryAWS/SimpleDB'
+require 'MockAWS/EC2'
+require 'MockAWS/S3'
+require 'MockAWS/SQS'
+require 'MockAWS/SimpleDB'
+require 'logger'
+
+# Make sure there is a Clock implementation.
+# If not, supply one.
+# Caller may supply a mock implementation instead of the standard one.
+begin
+  Clock.now
+rescue
+  require 'clock'
+end
+
+# Creates and caches EC2, SQS, and S3 interface implementations.
+# Can serve up AWS, SafeAWS, RetryAWS, or MockAWS interfaces.
+# Every time we create one, we reset the interfaces by forcing the
+# creation of new objects.
+# This is treated as a global.  It is initialized by calling "AwsContext.setup"
+# and then anyone can get a copy by calling "AwsContext.instance".  
+class AwsContext
+  private_class_method :new
+  @@instance = nil
+
+  def initialize(context, logger)
+    @context = context
+    @logger = logger
+    @ec2 = @sqs = @s3 = @sdb = nil
+  end
+
+  # Create a AwsContext.
+  #
+  # The context here can be one of 
+  #  * :aws -- use the standard AWS interface
+  #  * :safe -- use the SafeAWS interface
+  #  * :retry -- use the RetryAWS interface
+  #  * :mock -- use the MockAWS interface
+  #
+  # The interface is established the first time you use it.  The 
+  # returned interface is retained, and used for subsequent requests.
+  # Thus only one instance of each interface is created.
+  def AwsContext.setup(context, logger = nil)
+    @@instance = new(context, logger)
+  end
+
+  # Return the instance that was already created.
+  # If none created yet, then create AWS context.
+  def AwsContext.instance
+    if @@instance.nil?
+      setup(:aws)
+    end
+    @@instance
+  end
+
+  private
+
+  # Create an EC2 interface.
+  def create_ec2(*params)
+    case @context
+    when :aws
+     @ec2 = AWS::EC2.new(*params)
+    when :safe
+     @ec2 = SafeAWS::EC2.new(*params)
+    when :retry
+     @ec2 = RetryAWS::EC2.new(*params)
+    when :mock
+     @ec2 = MockAWS::EC2.new(*params)
+     @ec2.reset
+     @ec2.logger = @logger if @logger
+    end
+    @ec2
+  end
+
+  # Create an sqs interface.
+  def create_sqs(*params)
+    case @context
+    when :aws
+     @sqs = AWS::SQS.new(*params)
+    when :safe
+     @sqs = SafeAWS::SQS.new(*params)
+    when :retry
+     @sqs = RetryAWS::SQS.new(*params)
+    when :mock
+     @sqs = MockAWS::SQS.new(*params)
+     @sqs.reset
+     @sqs.logger = @logger if @logger
+    end
+    @sqs
+  end
+
+  # Create an S3 interface.
+  def create_s3(*params)
+    case @context
+    when :aws
+     @s3 = AWS::S3.new(*params)
+    when :safe
+     @s3 = SafeAWS::S3.new(*params)
+    when :retry
+     @s3 = RetryAWS::S3.new(*params)
+    when :mock
+     @s3 = MockAWS::S3.new(*params)
+     @s3.logger = @logger if @logger
+     @s3.reset
+    end
+    @s3
+  end
+
+  # Create a SimpleDB interface.
+  def create_sdb(*params)
+    case @context
+    when :aws
+     @sdb = AWS::SimpleDB.new(*params)
+    when :safe
+     @sdb = SafeAWS::SimpleDB.new(*params)
+    when :retry
+     @sdb = RetryAWS::SimpleDB.new(*params)
+    when :mock
+     @sdb = MockAWS::SimpleDB.new(*params)
+     @sdb.logger = @logger if @logger
+     @sdb.reset
+    end
+    @sdb
+  end
+
+  public
+
+  # Return an EC2 interface.  Create on if needed.
+  # Note that the parameters (if given) are only used if the interface is created.
+  def ec2(*params)
+    @ec2 || create_ec2(*params)
+  end
+
+  # Return an SQS interface.  Create on if needed.
+  # Note that the parameters (if given) are only used if the interface is created.
+  def sqs(*params)
+    @sqs || create_sqs(*params)
+  end
+
+  # Return a S3 interface.  Create on if needed.
+  # Note that the parameters (if given) are only used if the interface is created.
+  def s3(*params)
+    @s3 || create_s3(*params)
+  end
+
+  # Return a SimpleDB interface.  Create on if needed.
+  # Note that the parameters (if given) are only used if the interface is created.
+  def sdb(*params)
+    @sdb || create_sdb(*params)
+  end
+
+end

data/lib/basic_configuration.rb

@@ -0,0 +1,120 @@
+require 'inifile'
+require 'pp'
+
+# BasicConfiguration
+# Read an ini file and create a configuration.
+# The configuration contains the aws configuration only.
+
+class BasicConfiguration
+  attr_reader :aws
+
+  # Default list of config files to search for
+  DEFAULT_CONFIG_FILES = ['aws-config.ini', 'default-config.ini', 'config.ini']
+  # Search for config files in these directories
+  DEFAULT_SEARCH_PATH = [ ".", ENV['AWS_HOME'], ENV['HOME'], File.dirname(__FILE__)]
+
+  @@config_files = DEFAULT_CONFIG_FILES
+  @@search_path = DEFAULT_SEARCH_PATH
+
+  # Set up the list of config files that is read when looking for configurations.
+  # If this is not called, a default set of files is read.
+  def BasicConfiguration.setup_config_files(config_files = [])
+    @@config_files = config_files
+  end
+
+  # Set up the search path.
+  # If this is not called, a default path is used
+  def BasicConfiguration.setup_search_path(search_path = [])
+    @@search_path = search_path
+  end
+
+  # Add the given array of files to the list that we read.
+  def BasicConfiguration.add_config_files(config_files)
+    @@config_files |= config_files
+  end
+
+  # Add the following set of search paths to the ones that we use.
+  def BasicConfiguration.add_search_paths(search_paths)
+    @@search_path |= search_paths
+  end
+
+  # Create a config structure by reading the given config_filenames.
+  # If config files or search_paths are given, they apply only to this
+  # instance, but are retained for use by refresh.
+  # Create default aws config items based on environment varbiables, in
+  # case we can't find a config file.
+  def initialize(config_files = [], search_paths = [])
+    @config_files = @@config_files | config_files
+    @search_path = @@search_path | search_paths
+    refresh
+  end
+
+  # Read the configuration from the existing config file using
+  # the existing filenames.
+  # This is useful for reinitiaizing on a SIGTERM
+  def refresh
+    @aws = get_aws_from_environment
+    read_config
+  end
+
+  def keys
+    [ @aws[:aws_access_key], @aws[:aws_secret_key]]
+  end
+
+  private
+
+  # Looks up an environment variable, and returns the value, if there is one.
+  # If the caller asks for some environment variable that is not 
+  # present in the environment, then the default is returned.
+  def get_environment(var, default = nil)
+    ENV[var] || default
+  end
+
+
+  def get_aws_from_environment
+    # Get aws config from the environment
+    aws_user = get_environment('AWS_USER', 'aws')
+    key_file = File.join(ENV['HOME'], 'keys', "#{aws_user}-kp.pem")
+
+    # Set up our credentials from the environment
+    # These sere as a default to the credentials in the
+    # config.ini file.
+    { :aws_env =>  ENV['AWS_ENV'],
+      :aws_access_key => get_environment('AWS_ACCESS_KEY'),
+      :aws_secret_key => get_environment('AWS_SECRET_KEY'),
+      :aws_user => aws_user,
+      :aws_bucket => get_environment('AWS_BUCKET'),
+      :aws_key => get_environment('AWS_KEY', key_file)
+    }
+  end
+
+  # Read all the config files in order
+  def read_config
+    @config_files.each {|config_file| read(config_file)}
+  end
+
+  # Read the config file and store aws section.
+  def read(config_file)
+    ini = IniFile.load(find_on_search_path(config_file))
+    return nil unless ini
+    # Merge in the items from the 'aws' section, overriding the defaults.
+    # This needs to be first, so others can use it.
+    @aws.merge!(ini['aws'])
+    ini
+  end
+
+  # If filename starts with "/" then return it unchanged.
+  # Otherwise try various prefixes on it.
+  # Return the first one that is actually a file.
+  # Retun nil if the file does not exist at any of the places.
+  def find_on_search_path(filename)
+    return filename if filename[0..0] == File::SEPARATOR
+    @search_path.each do |prefix|
+      unless prefix.nil?
+        fn = File.join(prefix, filename)
+        return fn if File.exists?(fn)
+      end
+    end
+    nil
+  end
+end

data/lib/clock.rb

@@ -0,0 +1,10 @@
+
+# Used to wrap Time so it can be mocked
+
+class Clock < Time
+
+  def Clock.sleep(numeric = 0)
+    Kernel.sleep numeric
+  end
+  
+end

data/lib/factory.rb

@@ -0,0 +1,14 @@
+# This contains common factory methods.
+
+module Factory
+  # Given a class name as a string, create an instance of that class
+  # and initialize it with the given arguments.
+  def Factory.create_object_from_string(class_name, *args)
+    ObjectSpace.each_object(Class) do |x|
+      if x.name == class_name || x.name == "Cloudmaster::#{class_name}"
+        return x.new(*args)
+      end
+    end
+    nil
+  end
+end

data/lib/file_logger.rb

@@ -0,0 +1,36 @@
+# The default logger writes to a file descriptor.
+# This could be an actual file, or a stream such as STDOUT.
+class FileLogger
+  def initialize(file = nil, *args)
+    case
+    when file.nil?
+      @fd = STDOUT
+      @needs_close = false
+    when file.is_a?(String)
+      @fd = File.new(file, "a")
+      @needs_close = true
+    else
+      @fd = file
+      @needs_close = false
+    end
+  end
+
+  def puts(msg)
+    @fd.puts msg
+  end
+
+  def close
+    @fd.close if @needs_close
+  end
+
+  alias debug puts
+  alias info puts
+  alias notice puts
+  alias warning puts
+  alias err puts
+  alias error puts
+  alias alert puts
+  alias emerg puts
+  alias crit puts
+end
+

data/lib/inifile.rb

@@ -0,0 +1,148 @@
+#
+# This class represents the INI file and can be used to parse INI files.
+# Derived from IniFile gem, found on http://rubyforge.org/projects/inifile/
+#
+class IniFile
+
+  #
+  # call-seq:
+  #    IniFile.load( filename )
+  #    IniFile.load( filename, options )
+  #
+  # Open the given _filename_ and load the contents of the INI file.
+  # The following _options_ can be passed to this method:
+  #
+  #    :comment => ';'      The line comment character(s)
+  #    :parameter => '='    The parameter / value separator
+  #
+  def self.load( filename, opts = {} )
+    if filename 
+      new(filename, opts)
+    else
+      nil
+    end
+  end
+
+  def initialize( filename, opts = {} )
+    @fn = filename
+    @comment = opts[:comment] || '#'
+    @param = opts[:parameter] || '='
+    @ini = Hash.new {|h,k| h[k] = Hash.new}
+
+    @rgxp_comment = /^\s*$|^\s*[#{@comment}]/
+    @rgxp_section = /^\s*\[([^\]]+)\]/
+    @rgxp_param   = /^([^#{@param}]+)#{@param}(.*)$/
+
+    parse
+  end
+
+  #
+  # call-seq:
+  #    each {|section, parameter, value| block}
+  #
+  # Yield each _section_, _parameter_, _value_ in turn to the given
+  # _block_. The method returns immediately if no block is supplied.
+  #
+  def each
+    return unless block_given?
+    @ini.each do |section,hash|
+      hash.each do |param,val|
+        yield section, param, val
+      end
+    end
+    self
+  end
+
+  #
+  # call-seq:
+  #    each_section {|section| block}
+  #
+  # Yield each _section_ in turn to the given _block_. The method returns
+  # immediately if no block is supplied.
+  #
+  def each_section
+    return unless block_given?
+    @ini.each_key {|section| yield section}
+    self
+  end
+
+  #
+  # call-seq:
+  #    ini_file[section]
+  #
+  # Get the hash of parameter/value pairs for the given _section_.
+  #
+  def []( section )
+    return nil if section.nil?
+    @ini[section.to_s]
+  end
+
+  #
+  # call-seq:
+  #    has_section?( section )
+  #
+  # Returns +true+ if the named _section_ exists in the INI file.
+  #
+  def has_section?( section )
+    @ini.has_key? section.to_s
+  end
+
+  #
+  # call-seq:
+  #    sections
+  #
+  # Returns an array of the section names.
+  #
+  def sections
+    @ini.keys
+  end
+
+  private
+
+  def cleanup(str)
+    str = str.strip
+    first = str[0..0]; last = str[-1..-1]
+    str = str[1..-2] if first == last && (first == '"' || first == "'")
+  end
+  #
+  # call-seq
+  #    parse
+  #
+  # Parse the ini file contents.
+  #
+  def parse
+    return unless ::Kernel.test ?f, @fn
+    section = nil
+
+    ::File.open(@fn, 'r') do |f|
+      while line = f.gets
+        line = line.chomp
+
+        case line
+        # ignore blank lines and comment lines
+        when @rgxp_comment: next
+
+        # this is a section declaration
+        when @rgxp_section: section = @ini[$1.strip.downcase]
+
+        # otherwise we have a parameter
+        when @rgxp_param
+          begin
+	    val = $2.strip
+	    val = val[1..-2] if val[0..0] == "'" || val[-1..-1] == '"'
+            section[$1.strip.downcase.to_sym] = val
+          rescue NoMethodError
+            raise "Bad configuration - inifile parameter encountered before first section"
+          end
+
+        else
+          raise "Bad configuration -- inifile could not parse line '#{line}"
+        end
+      end
+    end
+  end
+
+end
+
+
+

data/lib/instance_logger.rb

@@ -0,0 +1,25 @@
+# The instance logger writes to a file in a given directory.
+# The file is determined by the instance_id in the message.
+# The file is opened and closed each time something is written.
+class InstanceLogger
+  def initialize(dir)
+    @directory = dir
+  end
+
+  def puts(instance, msg)
+    File.open(File.join(@directory, instance), "a") do |fd|
+      fd.puts msg
+    end
+  end
+
+  alias debug puts
+  alias info puts
+  alias notice puts
+  alias warning puts
+  alias err puts
+  alias error puts
+  alias alert puts
+  alias emerg puts
+  alias crit puts
+end
+

data/lib/logger_factory.rb

@@ -0,0 +1,38 @@
+require 'factory'
+require 'sys_logger'
+require 'string_logger'
+require 'file_logger'
+
+class LoggerFactory 
+  include Factory
+  @@filename = 'logfile'
+  @@program = 'unknown'
+
+  def LoggerFactory.setup(filename, program = 'unknown')
+    @@filename = filename
+    @@program = program
+  end
+
+  def LoggerFactory.create(type, *params)
+    filename = filename || @@filename
+    program = program || @@program
+    name = type.to_s
+    require name.downcase + '_logger'
+    class_name = name.capitalize + 'Logger'
+    logger = Factory.create_object_from_string(class_name, filename, program)
+    raise "Bad configuration -- no logger #{class_name}" unless logger
+    logger
+
+#    case type.to_s
+#    when 'syslog'
+#      SysLogger.new(program)
+#    when 'string'
+#      StringLogger.new
+#    when 'stdout'
+#      FileLogger.new(STDOUT, false)
+#    when 'file'
+#      FileLogger.new(File.new(filename, "a"), true)
+#    end
+  end
+end
+

data/lib/periodic.rb

@@ -0,0 +1,29 @@
+  #  Used to execute tasks periodically.
+  #  Execute at most once each interval (seconds).
+  #  If interval is < 0, never execute.
+  #  If interval is 0, execute every time.
+  class Periodic
+    # Create a periodic execution at this interval.
+    def initialize(interval)
+      @last_time = Clock.at(0)
+      @interval = interval
+    end
+
+    # Runs if it has not executed in the interval,.
+    # Skip otherwise.
+    def check    #expects a block
+      case
+      when @interval < 0
+        return
+      when @interval == 0
+        yield
+      when @interval > 0
+        now = Clock.now
+        if (@last_time + @interval) < now
+          yield
+          @last_time = now
+        end
+      end
+    end
+  end
+  

data/lib/string_logger.rb

@@ -0,0 +1,29 @@
+# Logs to a string.
+# This is useful in testing so that we can read back what
+# was put into the log, to see if it was what was expected.
+class StringLogger
+  def initialize(*params)
+    @log = StringIO.new("", "w")
+  end
+  def close
+  end
+
+  def puts(msg)
+    @log.puts(msg)
+  end
+
+  def string
+    @log.string
+  end
+
+  alias debug puts
+  alias info puts
+  alias notice puts
+  alias warning puts
+  alias err puts
+  alias error err
+  alias alert puts
+  alias emerg puts
+  alias crit puts
+end
+

data/lib/sys_logger.rb

@@ -0,0 +1,40 @@
+require 'syslog'
+
+# Logs using the syslog facility.
+class SysLogger
+  def initialize(dummy, program_name)
+    @log = Syslog.open(program_name)
+  end
+
+  def close
+  end
+  def puts(msg)
+    @log.info(msg)
+  end
+  def debug(msg)
+    @log.debug(msg)
+  end
+  def info(msg)
+    @log.info(msg)
+  end
+  def notice(msg)
+    @log.notice(msg)
+  end
+  def warning(msg)
+    @log.warning(msg)
+  end
+  def err(msg)
+    @log.err(msg)
+  end
+  def alert(msg)
+    @log.alert(msg)
+  end
+  def emerg(msg)
+    @log.emerg(msg)
+  end
+  def crit(msg)
+    @log.crit(msg)
+  end
+  alias error err
+end
+

data/lib/user_data.rb

@@ -0,0 +1,30 @@
+# This class loads user data.
+
+require 'open-uri'
+require 'yaml'
+
+class UserData
+  # Load user data
+  def self.load
+    begin
+      iid = open('http://169.254.169.254/latest/meta-data/instance-id').read(200)
+      user_data = open('http://169.254.169.254/latest/user-data').read(2000)
+      data = YAML.load(user_data)
+      aws = { :aws_env => data[:aws_env],
+            :aws_access_key => data[:aws_access_key],
+            :aws_secret_key => data[:aws_secret_key]
+	    }
+    rescue
+      # when running locally, use fake iid
+      iid = "unknown"
+      user_data = nil
+      aws = {}        
+    end
+    @@data = {:aws => aws, :iid => iid, :user_data => user_data}
+    @@data
+  end
+
+  def self.keys
+    [@@data[:aws][:aws_access_key], @@data[:aws][:aws_secret_key]] if @@data
+  end
+end

data/test/aws-config.ini

@@ -0,0 +1,9 @@
+#
+# EC2 Instance Manager Test Configuration
+[AWS]
+aws_env=test
+aws_access_key='test'
+aws_secret_key='test'
+aws_user=chayden
+aws_bucket=chayden
+aws_keypair=test