fairchild-poolparty 1.3.5 → 1.3.17

data/lib/cloud_providers/ec2/ec2_helpers.rb

@@ -1,12 +1,58 @@
 module CloudProviders
   module Ec2Helpers
     
+    # VOLUMES    
+    def attach_volume(instance_id, volume_id=next_unused_volume, device="/dev/sdh")
+      ec2.attach_volume(volume_id, instance_id, device)
+    end
+    
+    def next_unused_volume
+      if all_volumes.empty?
+        nil
+      else
+        available_volumes.first
+      end
+    end
+        
+    private
+    
+    def all_volumes
+      ebs_volumes.empty? ? [] : ec2.describe_volumes.select {|v| ebs_volumes.include?(v[:aws_id]) }
+    end
+    
+    def available_volumes
+      all_volumes.select {|v| v[:aws_status] == 'available' }
+    end
+    
+    def unavailable_volumes
+      all_volumes.reject {|v| available_volumes.include?(v) }
+    end
+    
+    public
+    
+    # SECURITY GROUPS
+    def security_groups(list=[])
+      ec2.describe_security_groups(list)
+    end
+    
+    public
+    
+    # ELASTIC IPS
+    
     # Associate an address with the instance using ec2
     # Get the next_unused_elastic_ip
     # and if there is one, associate the instance to the 
     # public ip
-    def associate_address()
-      raise StandardError.new('Not Implemented Yet')
+    def associate_address(instance_id)
+      new_ip = next_unused_elastic_ip
+      vputs("Assigning #{new_ip} to the ec2 instance #{instance_id}")
+      ec2.associate_address(instance_id, new_ip)
+      loop do
+        if describe_instance(:instance_id => instance_id).public_ip == new_ip
+          return new_ip
+        end
+        sleep 1
+      end
     end
     
     # Get the next usable elastic ip
@@ -17,13 +63,24 @@ module CloudProviders
     # intersection of the unused ips and those, find the first one available
     # and return that.
     def next_unused_elastic_ip
-      raise StandardError.new('Not Implemented Yet')
-      if elastic_ips.empty?
+      if unused_elastic_ips.empty?
         nil
       else
+        vputs("Found an unused elastic ip: #{unused_elastic_ips.first}")
+        unused_elastic_ips.first
       end
     end
-
+    
+    private
+    
+    def all_elastic_ips
+      elastic_ips.empty? ? [] : ec2.describe_addresses.map {|a| a[:public_ip]} & elastic_ips
+    end
+    
+    def unused_elastic_ips
+      all_elastic_ips.select {|i| i[:instance_id] == nil }
+    end
+    
     # Help create a keypair for the cloud
     # This is a helper to create the keypair and add them to the cloud for you
     # def create_keypair

data/lib/cloud_providers/ec2/ec2_instance.rb

@@ -64,7 +64,7 @@ module CloudProviders
       FileUtils.mkdir_p(cloud.tmp_path/ec2_dir) unless File.directory?(cloud.tmp_path/ec2_dir)
       run ["mkdir -p #{ec2_dir}"]
       # Save a yaml file of aws varibles and send to the instance
-      File.open(cloud.tmp_path/ec2_dir/'aws.yml', 'w') do |f|
+      File.open(cloud.tmp_path/"etc"/"poolparty"/'env.yml', 'w') do |f|
         f<<YAML::dump(cloud_provider.aws_hash(ec2_dir))  #TODO: don't save sensitive info in /tmp
       end
       # We scp these files directly to the instance so to reduce the risk of accidentally leaving them in an insecure location
@@ -85,6 +85,8 @@ module CloudProviders
                              /mnt
                              /proc
                              /sys
+                             /etc/ssh/ssh_host_*
+                             /etc/ssh/moduli
                              /etc/udev/rules.d/70-persistent-net.rules
                              /etc/udev/rules.d/z25_persistent-net.rules
                             )
@@ -110,11 +112,22 @@ module CloudProviders
       cmds << "mkdir -p #{opts[:destination]}/loop"
       cmds << "mount -o loop #{image_file} #{opts[:destination]}/loop"
       cmds << "rsync -ax #{rsync_excludes(opts[:exclude])} #{opts[:volume]}/ #{opts[:destination]}/loop/"
+      cmds << "if [[ -f /etc/init.d/ec2-ssh-host-key-gen ]]; then  chmod u+x /etc/init.d/ec2-ssh-host-key-gen ;fi"
       cmds << "umount #{opts[:destination]}/loop"
       self.run cmds
       image_file
     end
     
+    #TODO
+    # def bundle_and_register(opts={})
+    #   arch = 'uname'
+    #   image = make_image(opts)
+    #   'ec2-bundle-image' image
+    #   'ec2-upload-bundle'
+    #   'ec2-register-bundle'
+    #   return ami
+    # end
+    
   end
 
 end

data/lib/cloud_providers/vmware/vmware.rb

@@ -122,6 +122,9 @@ module CloudProviders
       end
       @vmx_file = "'#{o}'"
     end
+
+    def before_compile(args); end
+    def after_compile(args); end
     
   end
 end

data/lib/core/file.rb

@@ -0,0 +1,12 @@
+class File
+  class << self
+    alias_method :old_symlink, :symlink
+    def symlink(old_name, new_name)
+      begin
+        old_symlink(old_name, new_name)
+      rescue Errno::EEXIST
+        $stderr.puts "warning: symlinking #{old_name} -> #{new_name}. Already exists"
+      end
+    end
+  end
+end

data/lib/core/object.rb

@@ -68,7 +68,7 @@ class Object
   # MESSAGES
   # Debugging output helpers
   def vputs(m="")
-    puts "[INFO] -- #{m}" if verbose?
+    puts "[INFO] -- #{m.is_a?(String) ? m : m.inspect}" if verbose?
   end
   def dputs(m="")
     puts "[DEBUG] -- #{m.is_a?(String) ? m : m.inspect}" if debugging?

data/lib/dependency_resolvers/base.rb

@@ -89,14 +89,7 @@ module DependencyResolvers
           "#{obj.to_i}"
         end
       when String
-        case obj
-        when /^\d{4}$/
-          "#{obj}"
-        when /^\d{3}$/
-          "0#{obj}"
-        else
-          "\"#{obj}\""
-        end
+        "\"#{obj}\""
       when Proc
         obj.call # eh
       when Array
@@ -119,4 +112,4 @@ module DependencyResolvers
     
   end
   
-end
+end

data/lib/dependency_resolvers/chef.rb

@@ -109,10 +109,10 @@ module DependencyResolvers
       end
       
       def apply_meta_notifies(resource, add)
-        # The meta_notifies is a hash that looks like: {:file => [["pool_name", :reload]]}
+        # The meta_notifies is a hash that looks like: {:file => [["pool_name", :reload, :immediately]]}
         resource.meta_notifies.each do |ty, arr|
-          arr.each do |nm, action|
-            add << "  notifies :#{action}, resources(:#{chef_safe_resource(ty)} => \"#{nm}\")"
+          arr.each do |nm, action, at_time|
+            add << "  notifies :#{action}, resources(:#{chef_safe_resource(ty)} => \"#{nm}\"), :#{at_time}"
           end
         end
       end

data/lib/keypair.rb

@@ -7,10 +7,12 @@ class Keypair
   has_searchable_paths(:prepend_paths => [Dir.pwd, '/etc/poolparty/keys', "#{ENV["HOME"]}/.ssh/", "#{ENV["HOME"]}/.ec2/", ENV['EC2_CONFIG_DIR']])
   
   attr_accessor :filepath
+  attr_reader :extra_paths
   
   # Create a new key that defaults to id_rsa as the name. 
-  def initialize(fpath)
+  def initialize(fpath, extra_paths=[])
     @filepath = fpath
+    @extra_paths = extra_paths.map {|a| File.expand_path(a) }
     valid?
   end
   
@@ -27,10 +29,10 @@ class Keypair
   # Returns the full_filepath of the key. If a full filepath is passed, we just return the expanded filepath
   # for the keypair, otherwise query where it is against known locations
   def full_filepath
-    @full_filepath ||= if File.file?(::File.expand_path(filepath))
+    @full_filepath ||= if File.file?(File.expand_path(filepath))
       ::File.expand_path(filepath)
       else
-        search_in_known_locations(filepath)
+        search_in_known_locations(filepath, extra_paths)
       end
   end
   alias :to_s :full_filepath

data/lib/poolparty.rb

@@ -43,7 +43,9 @@ require "poolparty/pool_party_error"
     hash
     symbol
     proc
-    time).each do |lib|
+    time
+    file
+  ).each do |lib|
   require "core/#{lib}"
 end
 

data/lib/poolparty/base.rb

@@ -128,47 +128,70 @@ module PoolParty
     # to create edges on the graph
     def resources_graph(force=false)
       return @resources_graph if @resources_graph && !force
-      result = Digraph.new      
+      result = Digraph.new
       
       create_graph(resources, nil, result)
       
-      add_ordered_resources_to_result(resources, result)
-      
       @resources_graph = result
     end
     
-    # Add all the resources as edges of each other
-    def add_ordered_resources_to_result(resources, result)
-      arr_of_resources = resources.zip_offset(1)
-      
-      arr_of_resources.each do |first, second|
-        result.add_edge!(first, second) unless result.edge?(first, second) or result.edge?(second, first)
-        add_ordered_resources_to_result(first.resources, result)
-      end
-    end
-    
     # Create the graph of resources. Blow up if a resource isn't found
     # that is required. If it is found, add it as an edge to the 
     # dependency graph
     def create_graph(resources, on, result)
-      resources.each_with_index do |resource, idx|
+      # add_ordered_resources_to_result(without_dependencies, result)
+      first_layer_of_ordered_resources = resources_without_dependencies.zip_offset(1)
+      first_layer_of_ordered_resources.each do |first, second|
+        result.add_edge!(first, second) unless second.nil? or result.edge?(first, second) or result.edge?(second, first)
+      end
+      
+      resources_with_dependencies.each do |r|
         
-        resource.dependencies.each do |dep_type, deps_array|
+        r.dependencies.each do |dep_type, deps_array|
           deps_array.each do |dep_name|
             dep = get_resource(dep_type, dep_name)
-            raise PoolPartyError.create("ResourceNotFound", "A resource required for #{resource.has_method_name}(#{resource.name}) was not found: #{dep_type}(#{dep_name}). Please make sure you've specified this in your configuration.") unless dep
-            result.add_edge!(dep, resource, dep.name) unless result.edge?(dep, resource)
+            raise PoolPartyError.create("ResourceNotFound", "A resource required for #{dep_type}(#{dep_name}) was not found: #{dep_type}(#{dep_name}). Please make sure you've specified this in your configuration.") unless dep
+            
+            unless result.edge?(dep, r) and result.edge?(r, dep)
+              existing_connections = result.adjacent(dep)
+              existing_connections.each {|c| result.remove_edge!(r, c) }
+              
+              result.add_edge!(dep, r, dep.name)
+              
+              existing_connections.each {|c| result.add_edge!(dep, c) }
+            end
+            
           end
         end
-        
+      end
+      
+      all_resources.each_with_index do |resource, idx|
         if on
-          result.add_edge!(resource, on, resource.name) unless result.edge?(resource, on)
+          result.add_edge!(resource, on, resource.name) unless result.edge?(resource, on) or result.edge?(on, resource)
         else
-          result.add_vertex!(resource)
+          result.add_vertex!(resource) unless result.vertex?(resource)
         end
-        
-        create_graph(resource.resources, resource, result)
       end
+      
+      result
+    end
+    
+    # Add all the resources as edges of each other
+    def add_ordered_resources_to_result(resources, result)
+      arr_of_resources = resources.zip_offset(1)
+      
+      arr_of_resources.each do |first, second|
+        result.add_edge!(first, second) unless result.edge?(first, second) or result.edge?(second, first)
+        add_ordered_resources_to_result(first.resources, result)
+      end
+    end
+        
+    def resources_without_dependencies(r=all_resources)
+      r.reject {|a| !a.dependencies.empty? }
+    end
+    
+    def resources_with_dependencies(r=all_resources)
+      r - resources_without_dependencies(r)
     end
     
     # All the dependencies that are required by this resource

data/lib/poolparty/cloud.rb

@@ -27,7 +27,9 @@ module PoolParty
     end
     
     def before_compile
-      validate_all_resources
+      add_monitoring_stack_if_needed
+      
+      validate_all_resources unless ENV["POOLPARTY_NO_VALIDATION"]
     end
     
     # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
@@ -48,8 +50,8 @@ module PoolParty
     # returns an instance of Keypair
     # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
     # Or you can pass a full filepath
-    def keypair(n=nil)
-      @keypair ||= Keypair.new(n)
+    def keypair(n=nil, extra_paths=[])
+      @keypair ||= Keypair.new(n, extra_paths)
     end
     
     # Declare the CloudProvider for a cloud
@@ -58,10 +60,13 @@ module PoolParty
       return @cloud_provider if @cloud_provider
       self.cloud_provider_name = provider_symbol
       cloud_provider(o, &block)
+      cloud_provider.keypair(keypair.full_filepath)
     end
     
     # Cloud provider methods
-    def nodes(o={}); delayed_action {cloud_provider.nodes(o)}; end
+    def nodes(o={})
+       delayed_action {cloud_provider.nodes(o).collect{|n| n.cloud = self; n}}; 
+    end
     def run_instance(o={}); cloud_provider.run_instance(o);end
     def terminate_instance!(o={}); cloud_provider.terminate_instance!(o);end
     def describe_instances(o={}); cloud_provider.describe_instances(o);end
@@ -87,7 +92,7 @@ module PoolParty
     
     # 1.) Launches a new instance,
     # 2.) Waits for the instance to get an ip address
-    # 3.) Waits for port 22 to be open
+    # 3.) Waits for port ssh_port to be open
     # 4.) Calls call_after_launch_instance callbacks
     # 5.) Executes passed &block, if any
     # 6.) Returns the new instance object
@@ -98,7 +103,7 @@ module PoolParty
       instance.cloud = self
       @instance = instance
       #wait for an ip and then wait for ssh port, then configure instance
-      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(22, :timeout=>timeout)
+      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(ssh_port, :timeout=>timeout)
         callback :after_launch_instance
         instance.callback :before_bootstrap
         instance.bootstrap!
@@ -109,7 +114,7 @@ module PoolParty
         block.call(instance) if block
         instance
       else
-         raise StandardError.new("Instance port 22 not available")
+         raise StandardError.new("Instance port #{ssh_port} not available")
       end
       instance.refresh!
       instance
@@ -177,16 +182,31 @@ module PoolParty
       end
     end
     
+    # Add the monitoring stack
+    def add_monitoring_stack_if_needed
+      if monitors.size > 0
+        
+        run_in_context do
+          %w(collectd hermes).each do |m|
+            self.send m.to_sym
+          end
+        end
+        
+      end
+    end
+    
     # Take the cloud's resources and compile them down using 
     # the defined (or the default dependency_resolver, chef)
     def compile(caller=nil)
       callback :before_compile
+      cloud_provider.before_compile(self)
       FileUtils.mkdir_p tmp_path unless File.directory?(tmp_path)
       ddputs <<-EOE
 Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"} 
   number of resources: #{ordered_resources.size}
       EOE
       out = dependency_resolver.compile_to(ordered_resources, tmp_path/"etc"/"#{dependency_resolver_name}", caller)
+      cloud_provider.after_compile(self)
       callback :after_compile
       out
     end
@@ -215,14 +235,14 @@ Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}
     #     vote_for(:expand) if v > 0.8
     #   end
     def monitor(monitor_symbol, &block)
-      monitors[monitor_symbol] ||= PoolParty::Monitor.new(monitor_symbol, &block)
+      monitors[monitor_symbol.to_sym] ||= PoolParty::Monitor.new(monitor_symbol, &block)
     end
     
     # Run the monitor logic
     def run_monitor(monitor_name, value)
       mon = monitors[monitor_name.to_sym]
       if mon
-        mon.run(value.to_f)
+        mon.run(value)
       else
         "unhandled monitor"
       end
@@ -233,6 +253,14 @@ Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}
       @monitors ||= {}
     end
     
+    def monitor_format(mon_name, meth=nil, &block)
+      if monitors.has_key?(mon_name.to_sym)
+        monitors[mon_name.to_sym].format(meth, &block)
+      else
+        raise PoolPartyError.create("MonitorsFormattingError", "You created a monitor format for an unknown monitor. Please check and try again!")
+      end
+    end
+    
     ##### Internal methods #####
     # Methods that only the cloud itself will use
     # and thus are private
@@ -263,17 +291,23 @@ Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}
       if resources_graph.cyclic?
         cycles = []
         
-        resources_graph.edges.each do |edge|
-          if resources_graph.adjacent?(edge.source, edge.target) && resources_graph.adjacent?(edge.target, edge.source) && !cycles.include?(edge.source)
-            cycles << "#{edge.source.class}(#{edge.source.name}) depends on #{edge.target.class}(#{edge.target.name})"
-          end
+        cycles = resources_graph.find_cycle
+        cycle_string = cycles.map do |k,v|
+          "#{k} -> #{v}"
         end
+        
+        filepath = "/tmp"
+        format = "png"
+        dotpath = "#{filepath}/dot.#{format}"
+        resources_graph.write_to_graphic_file(format, filepath)
+        
+        `open #{dotpath}`
         msg =<<-EOE
       
         Your resource graph is cyclic. Two resources depend on each other, Cannot decide which resource
         to go first. Dying instead. Correct this and then try again.
       
-          #{cycles.join("\n          ")}
+          #{dotpath}
       
         Hint: You can see the resource graph by generating it with:
           cloud compile -g name
@@ -298,4 +332,25 @@ Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}
     end    
     
   end
-end
+end
+
+module GRATR
+  class Digraph
+    
+    # Crappy n*n
+    def find_cycle(from=self)
+      return [] unless cyclic?
+      cyclic_cycle = []
+      forward_edge = Proc.new {|e| }
+      back_edge    = Proc.new do |b| 
+        cyclic_cycle = dfs_tree_from_vertex(b)
+      end
+      from.dfs({ 
+       :forward_edge  => forward_edge,
+       :back_edge    => back_edge
+      })
+      cyclic_cycle
+    end
+    
+  end
+end