poolparty 1.2.2 → 1.3.1

data/lib/mixins/pinger.rb

@@ -0,0 +1,31 @@
+module PoolParty
+  module Pinger
+    module ClassMethods
+      # Test to see if the host has the available port open
+      # for response. 
+      def ping_port(host, port=22, retry_times=400)
+        connected = false
+        retry_times.times do |i|
+          begin
+            break if connected = TCPSocket.new(host, port).is_a?(TCPSocket)
+          rescue Exception => e
+            sleep(2)
+          end
+        end
+        connected
+      end
+    end
+  
+    module InstanceMethods
+      def ping_port(ip, port=22, retry_times=500);self.class.ping_port(ip, port, retry_times);end
+      def ping_port_and(ip, port=22, retry_times=500, &block)        
+        block.call if ping_port(ip, port, retry_times) && block
+      end
+    end
+  
+    def self.included(receiver)
+      receiver.extend         ClassMethods
+      receiver.send :include, InstanceMethods
+    end
+  end
+end

data/lib/mixins/printing.rb

@@ -0,0 +1,35 @@
+=begin rdoc
+  Binaries helper moduel
+=end
+module Printing
+  # print_msg
+  # Params
+  #   msg_array - Array of strings
+  # Usage:
+  #   msg = ["hello", "world"]
+  #   print_msg(msg)
+  def print_msg(msg_array)
+    arr = msg_array.map {|line| Colors.process(line) }
+    max_size = arr.sort {|a,b| a.size <=> b.size }.last.size
+    arr.each do |line|
+      puts line.ljust(max_size)
+    end
+    Colors.reset!
+  end
+  
+  # Ask for response
+  def are_you_sure?(msg)
+    puts msg
+    resp = STDIN.gets.chomp
+
+    case resp
+    when "Y"
+    when "yes"
+    when "y"
+      return true
+    else
+      return false
+    end
+  end
+  
+end

data/lib/mixins/searchable_paths.rb

@@ -0,0 +1,87 @@
+=begin rdoc
+  Abstracts out the searchable path for a resource such as a
+  template file, key, or clouds.rb
+  
+  NOTE: this is currently _only_ implemented on templates
+  (PoolParty::Resources::File) and not yet working on clouds.rb or key files.
+  These will eventually be refactored to use this class.
+=end
+
+module SearchablePaths
+  def self.included(mod)
+    mod.extend(ClassMethods)
+  end
+
+  module ClassMethods
+    # Specify that a particular class has methods for searchable paths.
+    # 
+    # Options:
+    # * <tt>:dirs</tt>: array of directories to look in *under* the search paths. (default: <tt>["/"]</tt>)
+    # * <tt>:dir</tt>: set the directory to look in *under* the search paths. Use either dir or dirs, not both. (default: +/+)
+    # * <tt>:paths</tt>: overwrite all default paths and set the paths to this array exactly
+    # * <tt>:append_paths</tt>:  append these paths to any existing paths
+    # * <tt>:prepend_paths</tt>: prepend these paths to any existing paths
+    def has_searchable_paths(opts={})
+      class_eval do
+        @searchable_paths_dirs = [opts[:dir]] if opts[:dir]
+        @searchable_paths_dirs = opts[:dirs]  if opts[:dirs]
+        @paths_override        = opts[:paths] if opts[:paths]
+        @paths_prepend         = opts[:prepend_paths] || []
+        @paths_append          = opts[:append_paths]  || []
+      end
+      extend SearchablePaths::SingletonMethods
+      include SearchablePaths::InstanceMethods
+    end
+    
+    def searchable_paths_dir;  @searchable_paths_dirs.first; end
+    def searchable_paths_dirs
+      @searchable_paths_dirs && @searchable_paths_dirs.size > 0 ? @searchable_paths_dirs : ["/"]
+    end
+   
+    # These are the default search paths in order:
+    # 
+    # * current working directory (Dir.pwd)
+    def default_paths
+      [
+        Dir.pwd
+      ]
+    end
+
+    # returns the full set of valid searchable paths, given the options
+    def searchable_paths
+      return @paths_override if @paths_override && @paths_override.size > 0
+      @searchable_paths ||= @paths_prepend + default_paths + @paths_append
+    end
+  end
+
+  module SingletonMethods
+  end
+
+  module InstanceMethods
+
+    # Searches for +filepath+ in the <tt>searchable_paths</tt> iff +filepath+
+    # doesn't exist. e.g. +filepath+ is interpreted *first* as an absolute
+    # path, if +filepath+ doesn't exist verbatim then it looks for the file
+    # in the searchable_paths.
+    # 
+    # Returns +nil+ if the file cannot be found.
+    def search_in_known_locations(filepath, additional_search_paths=[])
+      return filepath if File.exists?(filepath) # return the file if its an absolute path
+      additional_search_paths.each do |path|
+        full_path = File.expand_path(path / filepath)
+        return full_path if File.exists?(full_path)
+      end
+      self.class.searchable_paths.each do |path|
+        self.class.searchable_paths_dirs.each do |dir|
+          next if path.nil?
+          full_path = File.expand_path(path / dir / filepath)
+          return full_path if File.exists?(full_path)
+        end
+      end
+      nil
+    end
+    alias_method :find_file, :search_in_known_locations
+
+  end
+
+end

data/lib/poolparty/base.rb

@@ -0,0 +1,232 @@
+=begin rdoc
+  Base class for all PoolParty objects
+=end
+module PoolParty
+  
+  # Global storage for the current context_stack
+  def context_stack
+    $context_stack ||= []
+  end
+  
+  class Base
+    attr_reader :init_opts, :base_name
+    
+    include Parenting, Dslify
+    include SearchablePaths
+    include Callbacks
+    include Delayed
+    
+    default_options Default.default_options
+    
+    def initialize(opts={}, extra_opts={}, &block)
+      @init_block = block
+      @init_opts = compile_opts(opts, extra_opts)
+      
+      run_with_callbacks(init_opts, &block)
+      @base_name = self.name
+    end
+    
+    # Overloading the parent run_in_context
+    # First, push ourself to the stack
+    # then set all the variables given with the init
+    # and then eval the block and pop ourselves off the stack
+    # During runtime, the stack looks like
+    # 
+    # - self
+    #   - instance_eval block
+    # - stack
+    def run_in_context(o={}, &block)
+      proc = Proc.new do
+        set_vars_from_options(o)
+        instance_eval &block if block
+      end
+      super(&proc)
+    end
+    
+    # Run the block in the context of self
+    # and call the block after calling 
+    #   before_load
+    # and afterwards calling
+    #   after_loaded
+    def run_with_callbacks(o, &block)
+      run_in_context(o) do
+        callback :before_load, o, &block
+        instance_eval &block if block
+        callback :after_loaded, o, &block
+      end
+    end
+    
+    # Base callbacks for run_with_callbacks
+    # before_load
+    # This is called before the block is yielded
+    def before_load(o={}, &block)      
+    end
+    
+    # after_loaded
+    # This is run immediately after the block given
+    # on init is yielded
+    def after_loaded(o={}, &block)
+    end
+    
+    # Try to extract the name from the options
+    # Either the first parameter is a string or it is a hash
+    # if it is a hash, just merge the two hashes
+    # If it is a string, then merge it in as the name of the
+    # instance on the hash
+    def compile_opts(o={}, extra={})
+      case o
+      when Symbol, String
+        extra.merge(:name => o.to_s)
+      when NilClass
+        extra
+      else
+        extra.merge(o)
+      end
+    end
+    
+    # Validation checks
+    # if all of the validations pass, the object is considered valid
+    # the validations are responsible for raising a PoolPartyError (StandardError)
+    def valid?
+      validations.each {|validation| self.send(validation.to_sym) }
+    end
+    
+    # The array of validations that the instances must pass
+    # to be considered valid. These must be methods available on the instance
+    def validations
+      []
+    end
+    
+    # Ordered resources
+    # are the resources associated with this base
+    
+    # Resources are all the resources attached to this resource
+    def resources
+      @resources ||= []
+    end
+    
+    # Order the resources_graph using a top-sort iterator
+    def ordered_resources
+      resources_graph.topsort.to_a
+    end
+    
+    # Get a resource, based on it's type
+    # Used for INTERNAL use
+    def get_resource(ty, nm)
+      o = self.send "#{ty}s".to_sym
+      o.detect {|r| r.name == nm }
+    end
+    
+    def to_s
+      "#{self.respond_to?(:has_method_name) ? self.has_method_name : self.class.to_s.top_level_class}:#{name}"
+    end
+    
+    # Create a directed adjacency graph of each of the dependencies
+    # Add all the resources in order and their resources as well
+    # and then create the graph from the rest of the resources
+    # to create edges on the graph
+    def resources_graph(force=false)
+      return @resources_graph if @resources_graph && !force
+      result = Digraph.new
+      
+      add_ordered_resources_to_result(resources, result)
+      
+      create_graph(resources, nil, result)
+      
+      @resources_graph = result
+    end
+    
+    # Add all the resources as edges of each other
+    def add_ordered_resources_to_result(resources, result)
+      arr_of_resources = resources.zip_offset(1)
+      
+      arr_of_resources.each do |first, second|
+        result.add_edge!(first, second)
+        add_ordered_resources_to_result(first.resources, result)
+      end
+    end
+    
+    # Create the graph of resources. Blow up if a resource isn't found
+    # that is required. If it is found, add it as an edge to the 
+    # dependency graph
+    def create_graph(resources, on, result)
+      resources.each_with_index do |resource, idx|
+        
+        resource.dependencies.each do |dep_type, deps_array|
+          deps_array.each do |dep_name|
+            dep = get_resource(dep_type, dep_name)
+            raise PoolPartyError.create("ResourceNotFound", "A resource required for #{resource.has_method_name}(#{resource.name}) was not found: #{dep_type}(#{dep_name}). Please make sure you've specified this in your configuration.") unless dep
+            result.add_edge!(dep, resource, dep.name) unless result.edge?(dep, resource)
+          end
+        end
+        
+        if on
+          result.add_edge!(resource, on, resource.name) unless result.edge?(resource, on)
+        else
+          result.add_vertex!(resource)
+        end
+        
+        create_graph(resource.resources, resource, result)
+      end
+    end
+    
+    # All the dependencies that are required by this resource
+    # This is a hash of the dependencies required by the resource
+    def dependencies
+      @dependencies ||= {}
+    end
+    
+    def all_resources
+      resources.map do |res|
+        [res, res.all_resources ]
+      end.flatten
+    end    
+
+    # Write the cloud dependency graph
+    def output_resources_graph(fmt='png', dotfile=name, params={})
+        p = {
+          'bgcolor' => 'white',
+          'label'   => "#{name}",
+          'pad'     => '0.5',
+          'rankdir' => 'LR',
+          'overlap' => 'false',
+          'node_params' => {
+            'color' => '"#111111"',
+            'style' => "rounded"
+          }
+        }.merge(params)
+        resources_graph.write_to_graphic_file(fmt, dotfile, p)
+    end
+    
+    # The clouds.rb file
+    def clouds_dot_rb_file(n=nil); self.class.clouds_dot_rb_file(n); end
+    def self.clouds_dot_rb_file(n=nil)
+        Pool.clouds_dot_rb_file(n)
+    end
+    
+    # If the method is missing from ourself, check the Default
+    # class for the corresponding method.
+    # If the Default class has the method, inquire from the Default
+    # class, otherwise, call super
+    def method_missing(m,*a,&block)
+      if Default.respond_to?(m)
+        Default.send(m,*a,&block)
+      else
+        super
+      end
+    end
+    
+    def has_searchable_paths(opts={})
+      default_searchable_paths = [
+        Dir.pwd,
+        PoolParty::Default.poolparty_home_path,
+        PoolParty::Default.poolparty_src_path,
+        PoolParty::Default.poolparty_src_path/:lib/:poolparty,
+        PoolParty::Default.base_config_directory,
+        PoolParty::Default.remote_storage_path
+      ]
+      super(opts.merge(:paths => (opts.delete(:paths) || default_searchable_paths)))
+    end
+    
+  end
+end

data/lib/poolparty/cloud.rb

@@ -0,0 +1,246 @@
+module PoolParty  
+  class Cloud < DslBase
+    has_searchable_paths
+    
+    # Options we want on the output of the compiled script
+    # but want to take the options from the parent if they
+    # are nil on the cloud
+    default_options(
+      :minimum_instances        => 2,     # minimum_instances default
+      :maximum_instances        => 5,     # maximum_instances default
+      :minimum_runtime          => 3600,  # minimum_instances default: 1 hour
+      :contract_when            => nil,
+      :expand_when              => nil,
+      :cloud_provider_name      => :ec2,
+      :dependency_resolver_name => nil,
+      :os                       => nil,
+      :bootstrap_script         => nil
+    )
+    
+    # Define what gets run on the callbacks
+    # This is where we can specify what gets called
+    # on callbacks
+    #   parameters: cld, time
+    #   cld - the cloud the callback came from
+    #   callback - the callback called (i.e. :after_provision)
+    callback_block do |cld, callback|
+    end
+    
+    # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
+    # call and run instance_eval on the block and then call the after_create callback
+    def initialize(n, o={}, &block)
+      @cloud_name = n
+      @cloud_name.freeze
+      
+      # @init_block = block
+      @init_opts = compile_opts(o)
+      
+      @init_block = Proc.new do
+        super(n,o,&block)
+      end
+      
+    end
+    
+    # returns an instance of Keypair
+    # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
+    # Or you can pass a full filepath
+    def keypair(n=nil)
+      @keypair ||= Keypair.new(n)
+    end
+    
+    # Declare the CloudProvider for a cloud
+    #  Create an instance of the cloud provider this cloud is using
+    def using(provider_symbol, o={}, &block)
+      return @cloud_provider if @cloud_provider
+      self.cloud_provider_name = provider_symbol
+      cloud_provider(o, &block)
+    end
+    
+    # Cloud provider methods
+    def nodes(o={}); delayed_action {cloud_provider.nodes(o)}; end
+    def run_instance(o={}); cloud_provider.run_instance(o);end
+    def terminate_instance!(o={}); cloud_provider.terminate_instance!(o);end
+    def describe_instances(o={}); cloud_provider.describe_instances(o);end
+    def describe_instance(o={}); cloud_provider.describe_instance(o);end
+    
+    # Terminate all instances in the cloud
+    def terminate!
+      nodes.collect{|n| n.terminate! }
+    end
+    
+    # The actual cloud_provider instance
+    def cloud_provider(opts={}, &block)
+      return @cloud_provider if @cloud_provider
+      klass_name = "CloudProviders::#{cloud_provider_name}".classify
+      if provider_klass = CloudProviders.all.detect {|k| k.to_s == klass_name }
+        opts.merge!(:cloud => self, :keypair_name => self.keypair.basename)
+        @cloud_provider = provider_klass.new(dsl_options.merge(opts), &block)
+      else
+        raise PoolParty::PoolPartyError.create("UnknownCloudProviderError", "Unknown cloud_provider: #{cloud_provider_name}")
+      end
+      @cloud_provider
+    end
+    
+    # 1.) Launches a new instance,
+    # 2.) Waits for the instance to get an ip address
+    # 3.) Waits for port 22 to be open
+    # 4.) Calls call_after_launch_instance callbacks
+    # 5.) Executes passed &block, if any
+    # 6.) Returns the new instance object
+    def expand(opts={}, &block)
+      timeout = opts.delete(:timeout) || 300
+      callback :before_launch_instance
+      instance = cloud_provider.run_instance(opts)
+      instance.cloud = self
+      @instance = instance
+      #wait for an ip and then wait for ssh port, then configure instance
+      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(22, :timeout=>timeout)
+        callback :after_launch_instance
+        instance.callback :before_bootstrap
+        instance.bootstrap!
+        instance.callback :after_bootstrap
+        instance.callback :before_configure
+        instance.configure!
+        instance.callback :after_configure
+        block.call(instance) if block
+        instance
+      else
+         raise StandardError.new("Instance port 22 not available")
+      end
+      instance.refresh!
+      instance
+    end
+    
+    # Contract the cloud
+    def contract!(hsh={})
+      inst=nodes(hsh).last
+      inst.callback :before_terminate
+      inst.terminate!
+      inst.callback :after_terminate
+      inst
+    end
+    
+    # convenience method to loop thru all the nodes and configure them
+    def configure!(opts={}, threaded=true)
+      nodes.collect{|n| n.configure!}
+    end
+    
+    # Run command/s on all nodes in the cloud.
+    # Returns a hash of instance_id=>result pairs
+    def run(commands, opts={})
+      results = {}
+      threads = nodes.collect do |n|
+         Thread.new{ results[n.name] = n.run(commands, opts)  }
+      end
+      threads.each{ |aThread|  aThread.join }
+      results
+      # Serial implementation #
+      # nodes.inject({})do |results, n|
+      #   results[n.instance_id] = n.run(commands, opts)
+      #   results
+      # end
+    end
+    
+    # Temporary path
+    # Starts at the global default tmp path and appends the pool name
+    # and the cloud name
+    def tmp_path
+      Default.tmp_path / pool.name / name
+    end
+    
+    # The pool this cloud belongs to
+    def pool
+      parent
+    end
+    
+    # compile
+    
+    # Resolve with the dependency resolver
+    def resolve_with(a)
+      if DependencyResolvers.const_defined?(a.classify)        
+        dependency_resolver DependencyResolvers.module_eval("#{a.classify}")
+      else
+        raise PoolParty::PoolPartyError.create("DependencyResolverError", "Undefined dependency resolver: #{a}. Please specify one of the following: #{DependencyResolvers.all.join(", ")}")
+      end
+    end
+    
+    # Set the dependency resolver
+    def dependency_resolver(sym=nil)
+      @dependency_resolver ||= case sym
+      when :chef, nil
+        dsl_options[:dependency_resolver_name] = :chef
+        DependencyResolvers::Chef
+      end
+    end
+    
+    # Take the cloud's resources and compile them down using 
+    # the defined (or the default dependency_resolver, chef)
+    def compile(caller=nil)
+      FileUtils.mkdir_p tmp_path unless File.directory?(tmp_path)
+      ddputs("Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}")
+      dependency_resolver.compile_to(ordered_resources, tmp_path/"etc"/"#{dependency_resolver_name}", caller)
+    end
+    
+    # Get the os of the first node if it was not explicity defined, we'll assume they are
+    # all homogenous
+    def os(sym=nil)
+      if sym
+        dsl_options[:os] = sym
+      else
+        nodes.size > 0 ? nodes.first.os : dsl_options[:os]
+      end
+    end
+    alias :platform :os
+    
+    # The public_ip of the cloud is equivalent to the public_ip
+    # of the cloud's oldest node
+    def public_ip
+      nodes.first.public_ip
+    end
+    
+    ### MONITORS ###
+    # Create a new monitor on the cloud
+    # == Usage
+    #   monitor :cpu do |v|
+    #     vote_for(:expand) if v > 0.8
+    #   end
+    def monitor(monitor_symbol, &block)
+      monitors[monitor_symbol] ||= PoolParty::Monitor.new(monitor_symbol, &block)
+    end
+    
+    # Run the monitor logic
+    def run_monitor(monitor_name, value)
+      mon = monitors[monitor_name.to_sym]
+      if mon
+        mon.run(value.to_f)
+      else
+        "unhandled monitor"
+      end
+    end
+    
+    # Store the monitors in an array
+    def monitors
+      @monitors ||= {}
+    end
+    
+    ##### Internal methods #####
+    # Methods that only the cloud itself will use
+    # and thus are private
+    
+    # Form the cloud
+    # Run the init block with the init_opts
+    # on the cloud
+    # This is run after the cloud.rb file has been consumed
+    def form_clouds
+      run_with_callbacks(@init_opts, &@init_block)
+      loaded!
+    end
+    
+    def after_all_loaded
+      run_after_loaded do |b|
+        run_in_context(&b)
+      end
+    end
+    
+  end
+end

data/lib/poolparty/default.rb

@@ -0,0 +1,32 @@
+=begin rdoc
+  Default class
+  
+  Hangs on and defines defaults for PoolParty
+=end
+
+module PoolParty
+  class Default
+    include Dslify
+    
+    default_options(
+      :user => "root",
+      :os => :ubuntu,
+      :tmp_path => "/tmp/poolparty",
+      :poolparty_home_path => "#{ENV["HOME"]}/.poolparty",
+      :ec2_home => "#{ENV["HOME"]}/.ec2",
+      :poolparty_src_path => File.expand_path("#{::File.dirname(__FILE__)}/../../"),
+      :base_config_directory => "/etc/poolparty",
+      :remote_storage_path => "/var/poolparty",
+      :poolparty_log_path => "#{File.dirname(__FILE__)}/log/poolparty.log",
+      :verbose        => false,
+      :debug          => false,
+      :very_debugging => false
+    )
+        
+    # Method missing
+    def self.method_missing(m,*a,&block)
+      dsl_options.include?(m) ? dsl_options[m] : super
+    end
+    
+  end
+end

data/lib/poolparty/dsl_base.rb

@@ -0,0 +1,36 @@
+=begin rdoc
+  Dsl Base class for all cloud dsl methods
+  
+  Pool and Cloud both are taken from the DslBase so that both
+  carry the same dsl. The methods that apply to either/or
+  are defined in the respective classes
+=end
+
+module PoolParty
+  class DslBase < Base
+    
+    default_options(
+      :minimum_instances        => 2,
+      :maximum_instances        => 5
+    )
+            
+    # Set instances with a range or a number
+    # if passed with a hash, call nodes(hash) to return filtered list of 
+    # instances
+    def instances(arg)
+      case arg
+      when Range
+        minimum_instances arg.first
+        maximum_instances arg.last
+      when Fixnum
+        minimum_instances arg
+        maximum_instances arg
+      when Hash
+        nodes(arg)
+      else
+        raise PoolParty::PoolPartyError.create("DslMethodCall", "You must call instances with either a number, a range or a hash (for a list of nodes)")
+      end
+    end
+    
+  end
+end