fairchild-poolparty 1.2.12 → 1.3.5

data/lib/poolparty/base.rb

@@ -0,0 +1,233 @@
+=begin rdoc
+  Base class for all PoolParty objects
+=end
+module PoolParty
+  
+  # Global storage for the current context_stack
+  def context_stack
+    $context_stack ||= []
+  end
+  
+  class Base
+    attr_reader :init_opts, :base_name
+    
+    include Parenting, Dslify
+    include SearchablePaths
+    include Callbacks
+    include Delayed
+    
+    default_options Default.default_options
+    
+    def initialize(opts={}, extra_opts={}, &block)
+      @init_block = block
+      @init_opts = compile_opts(opts, extra_opts)
+      
+      run_with_callbacks(init_opts, &block)
+      @base_name = self.name
+    end
+    
+    # Overloading the parent run_in_context
+    # First, push ourself to the stack
+    # then set all the variables given with the init
+    # and then eval the block and pop ourselves off the stack
+    # During runtime, the stack looks like
+    # 
+    # - self
+    #   - instance_eval block
+    # - stack
+    def run_in_context(o={}, &block)
+      proc = Proc.new do
+        set_vars_from_options(:name => o[:name]) if o.has_key?(:name) # Name MUST be set first
+        set_vars_from_options(o)
+        instance_eval &block if block
+      end
+      super(&proc)
+    end
+    
+    # Run the block in the context of self
+    # and call the block after calling 
+    #   before_load
+    # and afterwards calling
+    #   after_loaded
+    def run_with_callbacks(o, &block)
+      run_in_context(o) do
+        callback :before_load, o, &block
+        instance_eval &block if block
+        callback :after_loaded, o, &block
+      end
+    end
+    
+    # Base callbacks for run_with_callbacks
+    # before_load
+    # This is called before the block is yielded
+    def before_load(o={}, &block)      
+    end
+    
+    # after_loaded
+    # This is run immediately after the block given
+    # on init is yielded
+    def after_loaded(o={}, &block)
+    end
+    
+    # Try to extract the name from the options
+    # Either the first parameter is a string or it is a hash
+    # if it is a hash, just merge the two hashes
+    # If it is a string, then merge it in as the name of the
+    # instance on the hash
+    def compile_opts(o={}, extra={})
+      case o
+      when Symbol, String
+        extra.merge(:name => o.to_s)
+      when NilClass
+        extra
+      else
+        extra.merge(o)
+      end
+    end
+    
+    # Validation checks
+    # if all of the validations pass, the object is considered valid
+    # the validations are responsible for raising a PoolPartyError (StandardError)
+    def valid?
+      validations.each {|validation| self.send(validation.to_sym) }
+    end
+    
+    # The array of validations that the instances must pass
+    # to be considered valid. These must be methods available on the instance
+    def validations
+      []
+    end
+    
+    # Ordered resources
+    # are the resources associated with this base
+    
+    # Resources are all the resources attached to this resource
+    def resources
+      @resources ||= []
+    end
+    
+    # Order the resources_graph using a top-sort iterator
+    def ordered_resources
+      resources_graph.topsort.to_a
+    end
+    
+    # Get a resource, based on it's type
+    # Used for INTERNAL use
+    def get_resource(ty, nm)
+      o = self.send "#{ty}s".to_sym
+      o.detect {|r| r.name == nm }
+    end
+    
+    def to_s
+      "#{self.respond_to?(:has_method_name) ? self.has_method_name : self.class.to_s.top_level_class}:#{name}"
+    end
+    
+    # Create a directed adjacency graph of each of the dependencies
+    # Add all the resources in order and their resources as well
+    # and then create the graph from the rest of the resources
+    # to create edges on the graph
+    def resources_graph(force=false)
+      return @resources_graph if @resources_graph && !force
+      result = Digraph.new      
+      
+      create_graph(resources, nil, result)
+      
+      add_ordered_resources_to_result(resources, result)
+      
+      @resources_graph = result
+    end
+    
+    # Add all the resources as edges of each other
+    def add_ordered_resources_to_result(resources, result)
+      arr_of_resources = resources.zip_offset(1)
+      
+      arr_of_resources.each do |first, second|
+        result.add_edge!(first, second) unless result.edge?(first, second) or result.edge?(second, first)
+        add_ordered_resources_to_result(first.resources, result)
+      end
+    end
+    
+    # Create the graph of resources. Blow up if a resource isn't found
+    # that is required. If it is found, add it as an edge to the 
+    # dependency graph
+    def create_graph(resources, on, result)
+      resources.each_with_index do |resource, idx|
+        
+        resource.dependencies.each do |dep_type, deps_array|
+          deps_array.each do |dep_name|
+            dep = get_resource(dep_type, dep_name)
+            raise PoolPartyError.create("ResourceNotFound", "A resource required for #{resource.has_method_name}(#{resource.name}) was not found: #{dep_type}(#{dep_name}). Please make sure you've specified this in your configuration.") unless dep
+            result.add_edge!(dep, resource, dep.name) unless result.edge?(dep, resource)
+          end
+        end
+        
+        if on
+          result.add_edge!(resource, on, resource.name) unless result.edge?(resource, on)
+        else
+          result.add_vertex!(resource)
+        end
+        
+        create_graph(resource.resources, resource, result)
+      end
+    end
+    
+    # All the dependencies that are required by this resource
+    # This is a hash of the dependencies required by the resource
+    def dependencies
+      @dependencies ||= {}
+    end
+    
+    def all_resources
+      resources.map do |res|
+        [res, res.all_resources ]
+      end.flatten
+    end    
+
+    # Write the cloud dependency graph
+    def output_resources_graph(fmt='png', dotfile=name, params={})
+        p = {
+          'bgcolor' => 'white',
+          'label'   => "#{name}",
+          'pad'     => '0.5',
+          'rankdir' => 'LR',
+          'overlap' => 'false',
+          'node_params' => {
+            'color' => '"#111111"',
+            'style' => "rounded"
+          }
+        }.merge(params)
+        resources_graph.write_to_graphic_file(fmt, dotfile, p)
+    end
+    
+    # The clouds.rb file
+    def clouds_dot_rb_file(n=nil); self.class.clouds_dot_rb_file(n); end
+    def self.clouds_dot_rb_file(n=nil);Pool.clouds_dot_rb_file(n);end
+    def clouds_dot_rb_dir(n=nil); self.class.clouds_dot_rb_dir(n);end
+    def self.clouds_dot_rb_dir(n=nil); Pool.clouds_dot_rb_dir(n);end
+    
+    # If the method is missing from ourself, check the Default
+    # class for the corresponding method.
+    # If the Default class has the method, inquire from the Default
+    # class, otherwise, call super
+    def method_missing(m,*a,&block)
+      if Default.respond_to?(m)
+        Default.send(m,*a,&block)
+      else
+        super
+      end
+    end
+    
+    def has_searchable_paths(opts={})
+      default_searchable_paths = [
+        Dir.pwd,
+        PoolParty::Default.poolparty_home_path,
+        PoolParty::Default.poolparty_src_path,
+        PoolParty::Default.poolparty_src_path/:lib/:poolparty,
+        PoolParty::Default.base_config_directory,
+        PoolParty::Default.remote_storage_path
+      ]
+      super(opts.merge(:paths => (opts.delete(:paths) || default_searchable_paths)))
+    end
+    
+  end
+end

data/lib/poolparty/cloud.rb

@@ -0,0 +1,301 @@
+module PoolParty  
+  class Cloud < DslBase
+    has_searchable_paths
+    
+    # Options we want on the output of the compiled script
+    # but want to take the options from the parent if they
+    # are nil on the cloud
+    default_options(
+      :minimum_instances        => 2,     # minimum_instances default
+      :maximum_instances        => 5,     # maximum_instances default
+      :minimum_runtime          => 3600,  # minimum_instances default: 1 hour
+      :contract_when            => nil,
+      :expand_when              => nil,
+      :cloud_provider_name      => :ec2,
+      :dependency_resolver_name => nil,
+      :os                       => nil,
+      :bootstrap_script         => nil
+    )
+    
+    # Define what gets run on the callbacks
+    # This is where we can specify what gets called
+    # on callbacks
+    #   parameters: cld, time
+    #   cld - the cloud the callback came from
+    #   callback - the callback called (i.e. :after_provision)
+    callback_block do |cld, callback|
+    end
+    
+    def before_compile
+      validate_all_resources
+    end
+    
+    # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
+    # call and run instance_eval on the block and then call the after_create callback
+    def initialize(n, o={}, &block)
+      @cloud_name = n
+      @cloud_name.freeze
+      
+      # @init_block = block
+      @init_opts = compile_opts(o)
+      
+      @init_block = Proc.new do
+        super(n,o,&block)
+      end
+      
+    end
+    
+    # returns an instance of Keypair
+    # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
+    # Or you can pass a full filepath
+    def keypair(n=nil)
+      @keypair ||= Keypair.new(n)
+    end
+    
+    # Declare the CloudProvider for a cloud
+    #  Create an instance of the cloud provider this cloud is using
+    def using(provider_symbol, o={}, &block)
+      return @cloud_provider if @cloud_provider
+      self.cloud_provider_name = provider_symbol
+      cloud_provider(o, &block)
+    end
+    
+    # Cloud provider methods
+    def nodes(o={}); delayed_action {cloud_provider.nodes(o)}; end
+    def run_instance(o={}); cloud_provider.run_instance(o);end
+    def terminate_instance!(o={}); cloud_provider.terminate_instance!(o);end
+    def describe_instances(o={}); cloud_provider.describe_instances(o);end
+    def describe_instance(o={}); cloud_provider.describe_instance(o);end
+    
+    # Terminate all instances in the cloud
+    def terminate!
+      nodes.collect{|n| n.terminate! }
+    end
+    
+    # The actual cloud_provider instance
+    def cloud_provider(opts={}, &block)
+      return @cloud_provider if @cloud_provider
+      klass_name = "CloudProviders::#{cloud_provider_name}".classify
+      if provider_klass = CloudProviders.all.detect {|k| k.to_s == klass_name }
+        opts.merge!(:cloud => self, :keypair_name => self.keypair.basename)
+        @cloud_provider = provider_klass.new(dsl_options.merge(opts), &block)
+      else
+        raise PoolParty::PoolPartyError.create("UnknownCloudProviderError", "Unknown cloud_provider: #{cloud_provider_name}")
+      end
+      @cloud_provider
+    end
+    
+    # 1.) Launches a new instance,
+    # 2.) Waits for the instance to get an ip address
+    # 3.) Waits for port 22 to be open
+    # 4.) Calls call_after_launch_instance callbacks
+    # 5.) Executes passed &block, if any
+    # 6.) Returns the new instance object
+    def expand(opts={}, &block)
+      timeout = opts.delete(:timeout) || 300
+      callback :before_launch_instance
+      instance = cloud_provider.run_instance(opts)
+      instance.cloud = self
+      @instance = instance
+      #wait for an ip and then wait for ssh port, then configure instance
+      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(22, :timeout=>timeout)
+        callback :after_launch_instance
+        instance.callback :before_bootstrap
+        instance.bootstrap!
+        instance.callback :after_bootstrap
+        instance.callback :before_configure
+        instance.configure!
+        instance.callback :after_configure
+        block.call(instance) if block
+        instance
+      else
+         raise StandardError.new("Instance port 22 not available")
+      end
+      instance.refresh!
+      instance
+    end
+    
+    # Contract the cloud
+    def contract!(hsh={})
+      inst=nodes(hsh).last
+      inst.callback :before_terminate
+      inst.terminate!
+      inst.callback :after_terminate
+      inst
+    end
+    
+    # convenience method to loop thru all the nodes and configure them
+    def configure!(opts={}, threaded=true)
+      nodes.collect{|n| n.configure! }
+    end
+    
+    # Run command/s on all nodes in the cloud.
+    # Returns a hash of instance_id=>result pairs
+    def run(commands, opts={})
+      results = {}
+      threads = nodes.collect do |n|
+         Thread.new{ results[n.name] = n.run(commands, opts)  }
+      end
+      threads.each{ |aThread|  aThread.join }
+      results
+      # Serial implementation #
+      # nodes.inject({})do |results, n|
+      #   results[n.instance_id] = n.run(commands, opts)
+      #   results
+      # end
+    end
+    
+    # Temporary path
+    # Starts at the global default tmp path and appends the pool name
+    # and the cloud name
+    def tmp_path
+      Default.tmp_path / pool.name / name
+    end
+    
+    # The pool this cloud belongs to
+    def pool
+      parent
+    end
+    
+    # compile
+    
+    # Resolve with the dependency resolver
+    def resolve_with(a)
+      if DependencyResolvers.const_defined?(a.classify)        
+        dependency_resolver DependencyResolvers.module_eval("#{a.classify}")
+      else
+        raise PoolParty::PoolPartyError.create("DependencyResolverError", "Undefined dependency resolver: #{a}. Please specify one of the following: #{DependencyResolvers.all.join(", ")}")
+      end
+    end
+    
+    # Set the dependency resolver
+    def dependency_resolver(sym=nil)
+      @dependency_resolver ||= case sym
+      when :chef, nil
+        dsl_options[:dependency_resolver_name] = :chef
+        DependencyResolvers::Chef
+      end
+    end
+    
+    # Take the cloud's resources and compile them down using 
+    # the defined (or the default dependency_resolver, chef)
+    def compile(caller=nil)
+      callback :before_compile
+      FileUtils.mkdir_p tmp_path unless File.directory?(tmp_path)
+      ddputs <<-EOE
+Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"} 
+  number of resources: #{ordered_resources.size}
+      EOE
+      out = dependency_resolver.compile_to(ordered_resources, tmp_path/"etc"/"#{dependency_resolver_name}", caller)
+      callback :after_compile
+      out
+    end
+    
+    # Get the os of the first node if it was not explicity defined, we'll assume they are
+    # all homogenous
+    def os(sym=nil)
+      if sym
+        dsl_options[:os] = sym
+      else
+        nodes.size > 0 ? nodes.first.os : dsl_options[:os]
+      end
+    end
+    alias :platform :os
+    
+    # The public_ip of the cloud is equivalent to the public_ip
+    # of the cloud's oldest node
+    def public_ip
+      nodes.first.public_ip
+    end
+    
+    ### MONITORS ###
+    # Create a new monitor on the cloud
+    # == Usage
+    #   monitor :cpu do |v|
+    #     vote_for(:expand) if v > 0.8
+    #   end
+    def monitor(monitor_symbol, &block)
+      monitors[monitor_symbol] ||= PoolParty::Monitor.new(monitor_symbol, &block)
+    end
+    
+    # Run the monitor logic
+    def run_monitor(monitor_name, value)
+      mon = monitors[monitor_name.to_sym]
+      if mon
+        mon.run(value.to_f)
+      else
+        "unhandled monitor"
+      end
+    end
+    
+    # Store the monitors in an array
+    def monitors
+      @monitors ||= {}
+    end
+    
+    ##### Internal methods #####
+    # Methods that only the cloud itself will use
+    # and thus are private
+    
+    # Form the cloud
+    # Run the init block with the init_opts
+    # on the cloud
+    # This is run after the cloud.rb file has been consumed
+    def form_clouds
+      run_with_callbacks(@init_opts, &@init_block)
+      loaded!
+    end
+    
+    def after_all_loaded
+      run_after_loaded do |b|
+        run_in_context(&b)
+      end
+    end
+    
+    def validate_all_resources
+      ddputs("Validating all the resources")
+      [:ensure_not_cyclic, :ensure_meta_fun_are_resources].each do |meth|
+        self.send meth
+      end
+    end
+    
+    def ensure_not_cyclic
+      if resources_graph.cyclic?
+        cycles = []
+        
+        resources_graph.edges.each do |edge|
+          if resources_graph.adjacent?(edge.source, edge.target) && resources_graph.adjacent?(edge.target, edge.source) && !cycles.include?(edge.source)
+            cycles << "#{edge.source.class}(#{edge.source.name}) depends on #{edge.target.class}(#{edge.target.name})"
+          end
+        end
+        msg =<<-EOE
+      
+        Your resource graph is cyclic. Two resources depend on each other, Cannot decide which resource
+        to go first. Dying instead. Correct this and then try again.
+      
+          #{cycles.join("\n          ")}
+      
+        Hint: You can see the resource graph by generating it with:
+          cloud compile -g name
+        
+        EOE
+        raise PoolPartyError.create("CyclicResourceGraphError", msg)
+      end
+    end
+    
+    def ensure_meta_fun_are_resources
+      resources.each do |res|
+        
+        if res.meta_notifies
+          res.meta_notifies.each do |ty, arr|
+            arr.each do |nm, action|
+              raise PoolPartyError.create("ResourceNotFound", "A resource required for #{ty}(#{nm}) was not found: #{ty}(#{nm}). Please make sure you've specified this in your configuration.") unless get_resource(ty, nm)
+            end
+          end
+        end
+        
+      end
+    end    
+    
+  end
+end