poolparty 0.0.4 → 0.2.6

data/lib/poolparty/modules/output.rb

@@ -0,0 +1,13 @@
+module PoolParty
+  module Output
+
+    def output(*args)
+      returning (@output ||= []) do |output|
+        args.each do |line|
+          (output ||= []) << line
+        end
+      end        
+    end      
+
+  end
+end

data/lib/poolparty/modules/pretty_printer.rb

@@ -0,0 +1,39 @@
+module PoolParty
+  module PrettyPrinter
+    
+    def pretty_print(prev="\t")
+      returning Array.new do |out|
+        out << pretty_name(prev, self)
+        out << pretty_options(prev, self)
+        
+        if self.respond_to?(:clouds)
+          clouds.each do |name, cl|
+            out << pretty_name(prev*2, cl)
+            out << pretty_options(prev*2, cl) #cl.pretty_print("#{prev}\t")
+          end
+        end
+        if self.respond_to?(:plugins)
+          out << "#{prev}\t\tPlugins"
+          out << "#{prev}\t\t" + plugins.map {|a| a}.join("\n")
+        end
+      end.join("\n")
+    end
+    
+    # Gather options on the object
+    # Do not print if the option is nil or empty.
+    # Also, don't show the option if the option is empty or the default option on the cloud    
+    def pretty_options(prev, o)
+      return "" unless o.respond_to?(:options)
+      print_options = (o.respond_to?(:parent) && o.parent && o.parent.respond_to?(:options)) ? 
+        (o.options.delete_if {|k,v| o.parent.options.has_key?(k) && o.parent.options[k] == o.options[k] && !o.options[k].nil? } ) : 
+        o.options
+      print_options = print_options.map {|k,v| [k, o.send(k.to_sym).to_s] }.inject({}) { |r,e| r[e[0]] = e[1] unless o.class.default_options[e[0]] == e[1] || e[1].nil? || e[1].empty?; r }
+      print_options.flush_out("#{prev}\t")
+    end
+    
+    def pretty_name(prev, o)
+      "#{prev}#{o.class.to_s.top_level_class.capitalize}: #{o.name if o.respond_to?(:name)}"
+    end
+    
+  end
+end

data/lib/{core/string.rb → poolparty/modules/s3_string.rb}

@@ -1,25 +1,4 @@
-class String
-  def hasherize(format=[])
-    hash = {}
-    i = 0
-    self.split(%r{[\n|\t|\s| ]+}).collect {|a| a.strip}.each do |f|
-      break unless format[i]
-      unless f == "" || f.nil?
-        hash[format[i]] = f
-        i+=1
-      end      
-    end
-    hash
-  end
-  def ^(h={})
-    self.gsub(/:([\w]+)/) {h[$1.to_sym] if h.include?($1.to_sym)}
-  end
-  def runnable
-    self.strip.gsub(/\n/, " && ")
-  end
-  def classify
-    self.capitalize
-  end
+module S3String
   def bucket_objects
     AWS::S3::Bucket.objects(self)
   end
@@ -46,4 +25,8 @@ class String
   def delete_bucket
     AWS::S3::Bucket.delete(self, :force => true) if bucket_exists?
   end
+end
+
+class String
+  include S3String
 end

data/lib/poolparty/monitors/base_monitor.rb

@@ -0,0 +1,16 @@
+=begin rdoc
+  Monitor class
+  
+  TODO: Fill this out
+=end
+module PoolParty
+  module Monitors
+    
+    class BaseMonitor
+      
+      
+      
+    end
+    
+  end
+end

data/lib/poolparty/net/remote.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + "/remoter_base"
+require File.dirname(__FILE__) + "/remoter"
+
+module PoolParty  
+  module Remote
+    
+    include PoolParty::Remote::Remoter
+    
+    def using(t)
+      if available_bases.include?(t.to_sym)
+        unless using_remoter? || t.nil?
+          self.class.send :attr_reader, :remote_base
+          mod = "#{t}".preserved_module_constant
+          
+          mod.send :include, PoolParty::Remote::RemoterBase
+          self.class.send :include, mod
+          self.extend mod
+                    
+          @remote_base = "#{t}".preserved_module_constant
+        end
+      else
+        puts "Unknown remote base" 
+      end
+    end
+    
+    def available_bases
+      remote_bases
+    end
+    
+    def using_remoter?
+      @remote_base ||= nil
+    end
+                
+  end  
+end

data/lib/poolparty/net/remote_bases/ec2.rb

@@ -0,0 +1,148 @@
+require 'rubygems'
+require 'EC2'
+
+class String
+  def convert_from_ec2_to_ip
+    self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
+  end
+end
+module PoolParty
+  module Ec2
+    def launch_new_instance!(num=1)
+      instance = ec2.run_instances(
+        :image_id => (ami || Base.ami),
+        :user_data => "",
+        :minCount => 1,
+        :maxCount => num,
+        :key_name => (keypair || Base.keypair),
+        :availability_zone => nil,
+        :size => "#{size || Base.size}")
+      begin
+        item = instance#.instancesSet.item
+        EC2ResponseObject.get_hash_from_response(item)
+      rescue Exception => e          
+      end
+    end
+    # Terminate an instance by id
+    def terminate_instance!(instance_id=nil)
+      ec2.terminate_instances(:instance_id => instance_id)
+    end
+    # Describe an instance's status
+    def describe_instance(id=nil)
+      describe_instances.select {|a| a[:name] == id}[0] rescue nil
+    end
+    def describe_instances
+      unless @describe_instances && !@describe_instances.empty?
+        @id = 0
+        @describe_instances = get_instances_description.each_with_index do |h,i|
+          if h[:status] == "running"
+            @name = @id == 0 ? "master" : "node#{@id}"
+            @id += 1
+          else
+            @name = "#{h[:status]}_node#{i}"
+          end
+          h.merge!({
+            :name => @name,
+            :hostname => h[:ip],
+            :ip => h[:ip].convert_from_ec2_to_ip
+          })
+        end
+      end
+      @describe_instances
+    end
+    # Get the s3 description for the response in a hash format
+    def get_instances_description
+      @cached_descriptions ||= EC2ResponseObject.get_descriptions(ec2.describe_instances).sort_by {|a| a[:launching_time]}
+    end
+    
+    # Help create a keypair for the cloud
+    # This is a helper to create the keypair and add them to the cloud for you
+    def create_keypair
+      return false unless keypair
+      unless ::File.exists?( new_keypair_path )
+        FileUtils.mkdir_p ::File.dirname( new_keypair_path )
+        Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+      end
+    end
+    # EC2 connections
+    def ec2
+      @ec2 ||= EC2::Base.new( :access_key_id => (access_key || Base.access_key), 
+                              :secret_access_key => (secret_access_key || Base.secret_access_key)
+                            )
+    end
+    
+    # Callback
+    def custom_install_tasks_for(o)
+      [
+        "# ec2 installation tasks",
+        "# Set hostname",
+        # "if [ -z $(grep -v '#' /etc/hosts | grep '#{o.name}') ]; then echo \"$(curl http://169.254.169.254/latest/meta-data/public-ipv4) #{o.name}\" >> /etc/hosts; fi",
+        "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
+        "hostname #{o.name}",
+        "echo #{o.name} > /etc/hostname"
+      ]
+    end
+    
+    def custom_configure_tasks_for(o)
+      [
+        "# ec2 configuration"
+      ]
+    end
+    
+    def reset!
+      @describe_instances = @cached_descriptions = nil
+    end
+  end
+  register_remote_base :Ec2
+end
+
+# Provides a simple class to wrap around the amazon responses
+class EC2ResponseObject
+  def self.get_descriptions(resp)      
+    rs = get_response_from(resp)
+    
+    # puts rs.methods.sort - rs.ancestors.methods
+    out = begin
+      if rs.respond_to?(:instancesSet)
+        [EC2ResponseObject.get_hash_from_response(rs.instancesSet.item)]
+      else
+        rs.collect {|r| 
+          if r.instancesSet.item.class == Array
+            r.instancesSet.item.map {|t| EC2ResponseObject.get_hash_from_response(t)}
+          else
+            [EC2ResponseObject.get_hash_from_response(r.instancesSet.item)]
+          end            
+        }.flatten.reject {|a| a.nil? }
+      end
+    rescue Exception => e
+      # Really weird bug with amazon's ec2 gem
+      rs.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil? } rescue []
+    end
+    
+    out
+  end
+  def self.get_response_from(resp)
+    begin
+      rs = resp.reservationSet.item unless resp.reservationSet.nil?
+      rs ||= resp.DescribeInstancesResponse.reservationSet.item
+      rs ||= rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      rs.reject! {|a| a.nil? || a.empty? }
+    rescue Exception => e
+    end
+    rs
+  end
+  def self.get_hash_from_response(resp)
+    begin
+      {
+        :instance_id => resp.instanceId,
+        :name => resp.instanceId, 
+        :ip => resp.dnsName || "not-assigned",
+        :status => resp.instanceState.name,
+        :launching_time => resp.launchTime,
+        :keypair => resp.keyName
+      }        
+    rescue Exception => e
+      nil
+    end      
+  end
+end

data/lib/poolparty/net/remote_instance.rb

@@ -0,0 +1,69 @@
+require File.dirname(__FILE__) + "/remoter"
+
+module PoolParty  
+  module Remote
+    
+    class RemoteInstance
+      include Remote
+      include Configurable
+      include CloudResourcer
+            
+      def initialize(opts, parent=self)
+        @parent = parent
+        set_vars_from_options(parent.options) if parent && parent.respond_to?(:options)
+        set_vars_from_options(opts) unless opts.nil? || opts.empty?
+        on_init
+      end
+      
+      # Callback
+      def on_init        
+      end
+      
+      # Is this remote instance the master?
+      def master?
+        name == "master"
+      end
+      
+      # The remote instances is only valid if there is an ip and a name
+      def valid?
+        !(ip.nil? || name.nil?)
+      end
+      
+      # Determine if the RemoteInstance is responding
+      def responding?
+        !responding.nil?
+      end
+      
+      # This is how we get the current load of the instance
+      # The approach of this may change entirely, but the usage of
+      # it will always be the same
+      def load
+        current_load ||= 0.0
+      end
+            
+      # Is this instance running?
+      def running?
+        !(status =~ /running/).nil?
+      end
+      # Is this instance pending?
+      def pending?
+        !(status =~ /pending/).nil?
+      end
+      # Is this instance terminating?
+      def terminating?
+        !(status =~ /shutting/).nil?
+      end
+      # Has this instance been terminated?
+      def terminated?
+        !(status =~ /terminated/).nil?
+      end
+      
+      # Printing. This is how we extract the instances into the listing on the 
+      # local side into the local listing file
+      def to_s
+        "#{name} #{ip}"
+      end
+    end
+    
+  end  
+end

data/lib/poolparty/net/remoter.rb

@@ -0,0 +1,224 @@
+=begin rdoc
+  This module is included by the remote module and defines the remoting methods
+  that the clouds can use to rsync or run remote commands
+=end
+require File.dirname(__FILE__) + "/../helpers/provisioner_base"
+
+module PoolParty
+  module Remote
+    module Remoter
+      def rsync_storage_files_to_command(remote_instance)
+        if remote_instance
+          "#{rsync_command} #{Base.storage_directory} #{remote_instance.ip}:#{Base.remote_storage_path}"
+        end
+      end
+      def run_command_on_command(cmd="ls -l", remote_instance=nil)
+        "#{ssh_command(remote_instance)} '#{cmd}'"
+      end
+      def ssh_command(remote_instance)
+        "#{ssh_string} #{remote_instance.ip}"
+      end
+      # Generic commandable strings
+      def ssh_string
+        (["ssh"] << ssh_array).join(" ")
+      end
+      # Array of ssh options
+      # Includes StrictHostKeyChecking to no
+      # Ssh with the user in Base
+      # And including the keypair_path
+      def ssh_array
+        ["-o StrictHostKeyChecking=no", "-l '#{Base.user}'", '-i "'+full_keypair_path+'"']
+      end
+      def rsync_command
+        "rsync --delete -azP --exclude cache -e '#{ssh_string}'"
+      end
+      # Open the cached local copy of the instances list and 
+      # create a new RemoteInstance from each line
+      def list_from_local
+        list_file = get_working_listing_file
+        if list_file
+          out = returning Array.new do |instances|
+            open(list_file).read.split("\n").each do |line|
+              instances << RemoteInstance.new(line)
+            end
+          end
+        else
+          out = list_from_remote(:cache => true)
+        end
+        return out
+      end      
+      # List the instances that are known from the remoter_base
+      # Create a RemoteInstance for each of the instances from the hash
+      # returned by the list of instances, write them to the cached file
+      # and then return the array of instances
+      def list_from_remote(options={})
+        out_array = get_remote_nodes
+        write_to_file( local_instances_list_file_locations.first, out_array.map{|a| a.to_s }.join("\n")) if options[:cache]
+        out_array
+      end
+      # Get the names of the nodes. Mainly used for puppet templating
+      def list_of_node_names(options={})
+        list_of_running_instances.collect {|ri| ri.name }
+      end
+      # An array of node ips. Mainly used for puppet templating
+      def list_of_node_ips(options={})
+        list_of_running_instances.collect {|ri| ri.ip }
+      end
+      def get_remote_nodes
+        returning Array.new do |instances|
+          list_of_instances(respond_to?(:keypair) ? keypair : nil).each do |h|
+            instances << PoolParty::Remote::RemoteInstance.new(h)
+          end
+        end
+      end
+      # Get the instance first instance file that exists on the system from the expected places
+      # denoted in the local_instances_list_file_locations
+      def get_working_listing_file
+        local_instances_list_file_locations.reject {|f| f unless File.file?(f) }.first
+      end
+      # Expected places for the instances.list to be located at on the machine
+      def local_instances_list_file_locations
+        [
+          "#{Base.storage_directory}/#{name}-instances.list",
+          "#{Base.base_config_directory}/#{name}-instances.list",
+          "~/.#{name}-instances.list",
+          "~/#{name}-instances.list",          
+          "#{name}-instances.list"
+        ]
+      end
+      
+      # List calculation methods
+      # 
+      # Are the minimum number of instances running?
+      def minimum_number_of_instances_are_running?
+        list_of_running_instances.size >= minimum_instances.to_i
+      end
+      # Can we shutdown an instance?
+      def can_shutdown_an_instance?
+        list_of_running_instances.size > minimum_instances.to_i
+      end
+      # Request to launch a number of instances
+      def request_launch_new_instances(num=1)
+        out = []
+        num.times {out << launch_new_instance!}
+        out
+      end
+      # Let's terminate an instance that is not the master instance
+      def request_termination_of_non_master_instance
+        inst = nonmaster_nonterminated_instances.last
+        terminate_instance!(inst.instance_id) if inst
+      end
+      # Can we start a new instance?
+      def can_start_a_new_instance?
+        maximum_number_of_instances_are_not_running?
+      end
+      # Are the maximum number of instances running?
+      def maximum_number_of_instances_are_not_running?
+        list_of_running_instances.size < maximum_instances.to_i
+      end
+      # Launch new instance while waiting for the number of pending instances
+      #  to be zero before actually launching. This ensures that we only
+      #  launch one instance at a time
+      def request_launch_one_instance_at_a_time
+        when_no_pending_instances { launch_new_instance! }
+      end
+      # A convenience method for waiting until there are no more
+      # pending instances and then running the block
+      def when_no_pending_instances(&block)
+        reset!
+        if list_of_pending_instances.size == 0
+          block.call if block
+        else
+          wait "5.seconds"
+          when_no_pending_instances(&block)
+        end
+      end
+      
+      # This will launch the minimum_instances if the minimum number of instances are not running
+      # If the minimum number of instances are not running and if we can start a new instance
+      def launch_minimum_number_of_instances        
+        if can_start_a_new_instance? && !minimum_number_of_instances_are_running?         
+          list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")
+          reset!
+          launch_minimum_number_of_instances unless minimum_number_of_instances_are_running?
+        end
+      end
+      # Launch the master and let the master handle the starting of the cloud
+      # We should only launch an instance if there are no pending instances, in the case 
+      # that the master has launched, but is still pending
+      # and if the master is not running AND we can start a new instance
+      # Then wait for the master to launch
+      def launch_and_configure_master!(testing=false)
+        request_launch_new_instances(1) if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running?
+        
+        when_no_pending_instances do
+          wait "10.seconds"
+          hide_output { Provisioner.provision_master(self, testing) }
+        end
+        
+      end
+      def is_master_running?
+        !list_of_running_instances.select {|a| a.name == "master"}.first.nil?
+      end
+      # Stub method for the time being to handle expansion of the cloud
+      def should_expand_cloud?(force=false)
+        force || false
+      end
+      # Stub method for the time being to handle the contraction of the cloud
+      def should_contract_cloud?(force=false)
+        force || false
+      end
+      # Expand the cloud
+      # If we can start a new instance and the load requires us to expand
+      # the cloud, then we should request_launch_new_instances
+      # Wait for the instance to boot up and when it does come back
+      # online, then provision it as a slave, this way, it is ready for action from the
+      # get go
+      def expand_cloud_if_necessary(force=false)
+        if can_start_a_new_instance? && should_expand_cloud?(force)
+          @out = request_launch_new_instances(1)
+          
+          reset!
+          when_no_pending_instances do
+            @ri = list_of_running_instances.last
+            PoolParty::Provisioner.provision_slave(@ri, self, !force)
+          end
+        end
+      end
+      # Contract the cloud
+      # If we can shutdown an instnace and the load allows us to contract
+      # the cloud, then we should request_termination_of_non_master_instance
+      def contract_cloud_if_necessary(force=false)
+        if can_shutdown_an_instance?          
+          request_termination_of_non_master_instance if should_contract_cloud?(force)
+        end
+      end
+      
+      # Rsync command to the instance
+      def rsync_storage_files_to(instance=nil)        
+        hide_output do
+          Kernel.system "#{rsync_storage_files_to_command(instance)}" if instance
+        end
+      end
+      # Take the rsync command and execute it on the system
+      # if there is an instance given
+      def run_command_on(cmd, instance=nil)        
+        Kernel.system "#{run_command_on_command(cmd, instance)}" if instance
+      end
+      
+      # Ssh into the instance given
+      def ssh_into(instance=nil)
+        Kernel.system "#{ssh_command(instance)}" if instance
+      end
+      # Find the instance by the number given
+      # and then ssh into the instance
+      def ssh_into_instance_number(num=0)
+        ssh_into( get_instance_by_number( num || 0 ) )
+      end
+
+      def self.included(receiver)
+        receiver.extend self
+      end
+    end
+  end
+end