auser-poolparty 1.2.12 → 1.3.0

data/lib/cloud_providers/cloud_provider.rb

@@ -0,0 +1,85 @@
+=begin rdoc
+  CloudProvider is the base class for cloud computing services such as Ec2, Eucalyptus - where your servers run.
+=end
+module CloudProviders
+  class CloudProvider
+    include Dslify
+    include Callbacks
+    
+    default_options(
+      :cloud        => nil,
+      :keypair_name => nil,
+      :image_id     => nil
+    )
+    
+    # All CloudProviders are added to the CloudProviders.all array
+    def self.inherited(subclass)
+      unless CloudProviders.all.include?(subclass)
+        CloudProviders.all << subclass
+      end
+    end
+    
+    def initialize(opts={}, &block)
+      set_vars_from_options(opts)
+      instance_eval(&block) if block
+    end
+    
+    # Required methods
+    # The next methods are required on all CloudProviderInstance types
+    # If your CloudProviderInstance type does not overwrite the following methods
+    # An exception will be raised and poolparty will explode into tiny little 
+    # pieces. Don't forget to overwrite these methods
+    
+    # Launch a new instance
+    def run_instance(o={})
+      raise StandardError.new("method_not_defined :launch_new_instance")
+    end
+    
+    # Terminate an instance by id
+    # Required params:
+    #   instance_id or instance_ids
+    def terminate_instance!(o={})
+      raise StandardError.new("method_not_defined :terminate_instance!")
+    end
+    
+    # Describe an instance's status.  Should return a hash like object
+    # Required keys are:
+    # :image_id
+    # :keypair_name
+    # :instance_id
+    # :status
+    def describe_instance(hash_of_criteria_to_select_instance_against)
+      raise StandardError.new("method_not_defined :describe_instance")
+    end
+    
+    # Get instances
+    # The instances must return an object responding to each
+    # Each yielded object must respond to [:status]
+    def describe_instances(o={})
+      raise StandardError.new("method_not_defined :describe_instances")
+    end
+    
+    # DSL and helpers
+    
+    # Returns an instance of Keypair
+    # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
+    # or you can pass a full filepath
+    def keypair(n=keypair_name)
+      @keypair ||= Keypair.new(n)
+      keypair_name @keypair.basename
+      @keypair
+    end
+    def keypair=(n)
+      @keypair = n
+    end
+    
+    # Nodes
+    # returns nodes from the describe_instances array
+    # These can be selected on by passing a hash
+    def nodes(hsh={})
+      results = describe_instances(:status=>'running').select_with_hash({:keypair_name => keypair.basename})
+      results.select_with_hash(hsh)
+    end
+    
+  end
+end

data/lib/cloud_providers/cloud_provider_instance.rb

@@ -0,0 +1,264 @@
+require 'socket'
+require 'timeout'
+
+module CloudProviders
+  
+  class CloudProviderInstance
+      include Dslify
+      # include Enumerable
+      include Connections
+      include Callbacks
+      
+      default_options(
+        :cloud        => nil, # The cloud this instance belongs to, set automaticly if node is created thru cloud expansion
+        :name         => nil, # Name of the remote instance (internal usage)
+        :internal_ip  => nil, # Internal ip of the remote instance
+        :public_ip    => nil,
+        :dns_name     => nil,
+        :status       => nil, # Status of the remote instance
+        :launch_time  => nil,
+        :keypair_name => nil,
+        :cloud_name   => nil
+      )
+      
+      def initialize(opts={}, &block)
+        self.cloud_name= opts[:cloud].name if opts[:cloud]
+        set_vars_from_options(opts)
+        instance_eval(&block) if block
+        loaded
+      end
+      
+      # Returns an instance of Keypair
+      # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
+      # or you can pass a full filepath
+      def keypair(n=keypair_name)
+        cloud_provider.keypair(n)
+      end
+      
+      def cloud_provider(opts={}, &block)
+        raise StandardError.new("cloud_provider has not been implemented for this CloudProviderInstance ")
+      end
+      
+      # CLOUD PROVIDER METHODS
+      
+      # Bootstrap self.  Bootstrap runs as root, even if user is set
+      def bootstrap!(opts={})        
+         if !bootstrapped? || opts[:force]
+           old_user = user
+           @user = "root"
+           opts[:os] ||= os || determine_os
+           if cloud && cloud.bootstrap_script
+             opts[:filename] = cloud.bootstrap_script
+           end
+          script_file = Provision::Bootstrapper.bootstrap_script(opts)
+          scp(:source => script_file, :destination => "/tmp")
+          output = run("chmod +x /tmp/determine_os.sh; /bin/sh /tmp/#{File.basename(script_file)}")
+          @user = old_user
+          output.chomp if output
+        end
+      end
+      
+      # Configure the node
+      def configure!(opts={})
+        bootstrap! unless bootstrapped?
+        raise StandardError.new("You must pass in a cloud to configure an instance") unless cloud
+        cloud.compile(self)
+        scp(:source       => keypair.full_filepath, 
+            :destination  => "/etc/poolparty/keys/#{keypair.basename}")
+        script_file = Provision::Bootstrapper.configure_script(cloud, os)
+        
+        FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty" unless File.directory?(cloud.tmp_path/"etc"/"poolparty")
+        FileUtils.cp script_file, cloud.tmp_path/"etc"/"poolparty"
+        
+        rsync(:source => cloud.tmp_path/"*", :destination => "/")
+        run("chmod +x /etc/poolparty/#{File.basename(script_file)}; /bin/sh /etc/poolparty/#{File.basename(script_file)}").chomp
+        run(cloud.dependency_resolver.compile_command)
+      end
+      
+      # Terminate self
+      def terminate!
+        cloud_provider.terminate_instance!(:instance_id => instance_id)
+      end
+      
+      # get the os, if it's not declared
+      def os(sym=nil)
+        if sym
+          dsl_options[:os] = sym
+        else
+          dsl_options[:os] ||= determine_os.to_sym
+        end
+      end
+      alias :platform :os  # Chef uses platform, aliased for conveneince
+      
+      # Determine the os
+      # Default to ubuntu
+      # Send the determine_os.sh script to the node and run it remotely
+      def determine_os
+        scp(:source => Provision::Bootstrapper.determine_os_script, :destination => "/tmp")
+        o = run("chmod +x /tmp/determine_os.sh; /bin/sh /tmp/determine_os.sh").chomp
+        o.empty? ? :ubuntu : o
+      end
+      
+      # Determine if the node is bootstrapped
+      def bootstrapped?
+        # @bootstrapped ||= !run('if [ -f /var/poolparty/bootstrapped ]; then echo "YES"; fi').match(/YES/).nil?
+        @bootstrapped ||= !run('if [ -f /var/poolparty/bootstrapped ]; then echo "YES"; fi').chomp.empty? || false
+      end
+      
+      # Wait for port
+      # Test if the port is open and catch failures in the connection
+      # Options
+      #   public_ip || default public_ip
+      #   retry_times || 5
+      def wait_for_port(port, opts={})
+        ip          = opts.delete(:public_ip)   || public_ip
+        retry_times = opts.delete(:retry_times) || 10
+        pause_time  = opts.delete(:pause_time)  || 1
+        
+        retry_times.times do |i| 
+          if is_port_open?(ip, port, opts)
+            return true
+          else
+            sleep pause_time
+          end
+        end
+        false
+      end
+      
+      # Wait for a public ip to be assigned, refreshing the instance data from the cloud provider on each query
+      # Default timeout value of 60 seconds, can be overriden by passing {:timeout=>seconds}
+      def wait_for_public_ip(timeout=60)
+        ddputs("Waiting for public ip")
+        begin
+          Timeout::timeout(timeout) do
+            loop do
+              self.refresh!
+              ddputs("After refreshing, the public_ip is: #{public_ip}")
+              return public_ip if valid_ip?(public_ip)
+              print '.'
+              sleep 2
+            end
+          end
+        rescue Timeout::Error
+          return false
+        end
+      end
+      
+      def terminate!
+        cloud_provider.terminate_instance!(:instance_id=>self.instance_id).first
+      end
+      
+      # Refresh the node with fresh data from the cloud provider.
+      # This is often usefully to update a recently launched instance, in case you want to trigger new behavior once the state changes ot 'running' and an ip is assigned
+      # Added rescue, but not sure if this is a proper fix yet. Will need to test on live instances to ensure
+      def refresh!
+        dsl_options = cloud_provider.describe_instance(:instance_id => self.instance_id).dsl_options rescue {}
+        self.dsl_options.merge!(dsl_options)
+        self
+      end
+      
+      ## hash like methods
+      def each
+        dsl_options.each{ |k,v| yield k,v }
+      end
+      
+      def [](k)
+        if dsl_options.has_key? k
+          dsl_options[k]
+        else
+          nil
+        end
+      end
+      
+      def []=(k,v)
+        dsl_options[k] = v
+      end
+      
+      def has_key?(key)
+        dsl_options.has_key?(key)
+      end
+      
+      def keys
+        dsl_options.keys
+      end
+         
+      def values
+        dsl_options.values
+      end
+      
+      def to_hash
+        dsl_options
+      end
+      ##end of hash like methods
+      
+      # Is this instance running?
+      def running?
+        !(status =~ /running/).nil?
+      end
+      # Is this instance pending?
+      def pending?
+        !(status =~ /pending/).nil?
+      end
+      # Is this instance terminating?
+      def terminating?
+        !(status =~ /shutting/).nil?
+      end
+      # Has this instance been terminated?
+      def terminated?
+        !(status =~ /terminated/).nil?
+      end
+      
+      def elapsed_runtime
+        Time.now.to_i - launching_time.to_time.to_i
+      end
+      
+      # Callbacks
+      def loaded
+      end
+      
+      def before_bootstrap
+      end
+      def after_bootstrap
+      end
+      def before_configure
+      end
+      def after_configure
+      end
+      
+      # The instances is only valid if there is an internal_ip and a name
+      def valid?
+        (internal_ip.nil? || name.nil?) ? false : true
+      end
+      
+      private
+      
+      def valid_ip?(ip)
+        ip && ip != '0.0.0.0' && ip != ''
+      end
+
+      # Test for open port by opening a socket
+      # on the ip and closing the socket
+      def is_port_open?(ip, port, opts={})
+        timeout = opts[:timeout] || 1
+        begin
+          Timeout::timeout(timeout) do
+            begin
+              s = TCPSocket.new(ip, port)
+              s.close
+              ddputs("Connected to #{ip}:#{port} - Port is open and good to go")
+              return true
+              puts ','
+            rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+              ddputs("Port #{port} on #{ip} is not accessible (yet)")
+              return false
+            end
+          end
+        rescue Timeout::Error
+        end
+        ddputs("Port #{port} on #{ip} is not accessible")
+        return false
+      end
+      
+    end
+    
+end

data/lib/cloud_providers/connections.rb

@@ -0,0 +1,99 @@
+require "open3"
+
+module  CloudProviders
+  module Connections
+    
+    def user(n=nil)
+      if n.nil? 
+        @user ||= 'root'
+      else
+        @user = n
+      end
+    end
+    
+    # hostname or ip to use when running remote commands
+    def host(n=nil)
+      if n.nil? 
+        @host ||= dns_name
+      else
+        @host = n
+      end
+    end
+    
+    def run(commands, o={})
+      ssh(commands)
+    end
+    
+    # Simply shell out and call ssh, simple, reliable and fewest dependencies, but slow
+    def ssh( commands=[], extra_ssh_ops={})
+      commands = commands.compact.join(' && ') if commands.is_a?(Array)
+      cmd_string = "ssh #{user}@#{host} #{ssh_options(extra_ssh_ops)} "
+      if commands.empty?
+        #TODO: replace this with a IO.popen call with read_nonblocking to show progress, and accept input
+        Kernel.system(cmd_string)
+      else
+        system_run(cmd_string+"'#{commands}'")
+      end
+    end
+    
+    # Take a hash of options and join them into a string, combined with default options.
+    # Default options are -o StrictHostKeyChecking=no -i keypair.full_filepath -l user
+    # {'-i'=>'keyfile, '-l' => 'fred' } would become
+    # "-i keyfile -o StrictHostKeyChecking=no -i keypair.to_s -l fred"
+    def ssh_options(opts={})
+      o = {"-i" => keypair.full_filepath,
+           "-o" =>"StrictHostKeyChecking=no"
+           }.merge(opts)
+      o.collect{ |k,v| "#{k} #{v}"}.join(' ')
+    end
+    
+    def rsync( opts={} )
+      raise StandardError.new("You must pass a :source=>uri option to rsync") unless opts[:source]
+      destination_path = opts[:destination] || opts[:source]
+      rsync_opts = opts[:rsync_opts] || '-va'
+      cmd_string =  "rsync -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
+      out = system_run(cmd_string)
+      out
+    end
+    
+    def scp(opts={})
+      source = opts[:source]
+      destination_path = opts[:destination] || opts[:source]
+      raise StandardError.new("You must pass a local_file to scp") unless source
+      scp_opts = opts[:scp_opts] || ""
+      cmd_string = "scp #{ssh_options(scp_opts)} #{source} #{user}@#{host}:#{destination_path}"
+      out = system_run(cmd_string)
+      out
+    end
+    
+    private
+    
+    # Execute command locally.
+    # This method is mainly broken out to ease testing in the other methods
+    # It opens the 3 IO outputs (stdin, stdout, stderr) and print the output out
+    # as the command runs, unless the quiet option is passed in
+    def system_run(cmd, o={})
+      opts = {:quiet => false, :sysread => 1024}.merge(o)
+      buf = ""
+      ddputs("Running command: #{cmd}")
+      Open3.popen3(cmd) do |stdout, stdin, stderr|
+        begin
+          while (chunk = stdin.readpartial(opts[:sysread]))
+            buf << chunk
+            unless chunk.nil? || chunk.empty?
+              $stdout.write(chunk) if debugging? || verbose?
+            end
+          end
+          err = stderr.readlines
+          $stderr.write_nonblock(err)
+        rescue SystemCallError => error
+          $stderr.write_nonblock(stderr)
+        rescue EOFError => error
+           # nothing
+        end
+      end
+      buf
+    end
+    
+  end
+end

data/lib/cloud_providers/ec2/ec2.rb

@@ -0,0 +1,197 @@
+=begin rdoc
+  EC2 CloudProvider
+  This serves as the basis for running PoolParty on Amazon's ec2 cloud.
+=end
+require "openssl"
+if OpenSSL::OPENSSL_VERSION_NUMBER < 0x00908000
+  warn "the ec2 cloud provider may not work with your version of ruby and OpenSSL.  Consider upgrading if you encoutner authentication errors."
+end
+begin
+  require 'right_aws'
+rescue LoadError
+  puts <<-EOM
+Error: In order to use ec2, you need to install the right_aws gem
+
+Ec2 is the default cloud provider for PoolParty. If you intend on using
+a different provider, specify it with:
+
+using :provider_name
+EOM
+end
+
+require "#{File.dirname(__FILE__)}/ec2_response"
+require "#{File.dirname(__FILE__)}/ec2_instance"
+
+module CloudProviders
+  class Ec2 < CloudProvider
+    
+    # Set the aws keys from the environment, or load from /etc/poolparty/aws.yml if the environment variable is not set
+    def self.default_access_key
+      ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key]
+    end
+    
+    def self.default_secret_access_key
+      ENV['EC2_SECRET_KEY'] || load_keys_from_file[:access_key]
+    end
+    
+    def self.default_private_key
+      ENV['EC2_PRIVATE_KEY'] || load_keys_from_file[:private_key]
+    end
+    
+    def self.default_cert
+      ENV['EC2_CERT'] || load_keys_from_file[:cert]
+    end
+    
+    def self.default_user_id
+      ENV['EC2_USER_ID'] || load_keys_from_file[:user_id]
+    end
+    
+    def self.default_ec2_url
+      ENV['EC2_URL'] || load_keys_from_file[:ec2_url]
+    end
+    
+    def self.default_s3_url
+      ENV['S3_URL'] || load_keys_from_file[:s3_url]
+    end
+    
+    def self.default_cloud_cert
+     ENV['CLOUD_CERT'] || ENV['EUCALYPTUS_CERT'] || load_keys_from_file[:cloud_cert]
+    end
+    
+    # Load the yaml file containing keys.  If the file does not exist, return an empty hash
+    def self.load_keys_from_file(filename='/etc/poolparty/ec2/aws.yml', caching=true)
+      return @aws_yml if @aws_yml && caching==true
+      return {} unless File.exists?(filename)
+      @aws_yml = YAML::load( open(filename).read )
+    end
+    
+    default_options({
+        :image_id               => 'ami-bf5eb9d6',
+        :instance_type          => 'm1.small',
+        :addressing_type        => "public",
+        :availability_zone      => "us-east-1a",
+        :security_group         => ["default"],
+        :user_id                => default_user_id,
+        :private_key            => default_private_key,
+        :cert                   => default_cert,
+        :cloud_cert             => default_cloud_cert,
+        :access_key             => default_access_key,
+        :secret_access_key      => default_secret_access_key,
+        :ec2_url                => default_ec2_url,
+        :s3_url                 => default_s3_url,
+        :min_count              => 1,
+        :max_count              => 1,
+        :user_data              => '',
+        :addressing_type        => nil,
+        :kernel_id              => nil,
+        :ramdisk_id             => nil,
+        :availability_zone      => nil,
+        :block_device_mappings  => nil,
+        :elastic_ips            => nil, # An array of the elastic ips
+        :ebs_volume_id          => nil  # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
+      })
+      
+      
+    def ec2(o={})
+      @ec2 ||= Rightscale::Ec2.new(access_key, secret_access_key, o.merge(:logger => PoolParty::PoolPartyLog))
+    end
+    
+    # Start a new instance with the given options
+    def run_instance(o={})
+      set_vars_from_options o
+      raise StandardError.new("You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}") if !keypair_name
+      response_array = ec2(o).run_instances(image_id,
+                                      min_count,
+                                      max_count,
+                                      security_group,
+                                      keypair.basename,
+                                      user_data,
+                                      addressing_type,
+                                      instance_type,
+                                      kernel_id,
+                                      ramdisk_id,
+                                      availability_zone,
+                                      block_device_mappings
+                                      )
+      instances = response_array .collect do |aws_response_hash|
+        Ec2Instance.new( Ec2Response.pp_format(aws_response_hash).merge(o) )
+      end
+      #FIXME: This needs to deal with the case when an array is returned if max_instances > 1
+      instances.first
+    end
+    
+    # Will select the first instance matching the provided criteria hash
+    def describe_instance(hash_of_criteria_to_select_instance_against)
+      describe_instances(hash_of_criteria_to_select_instance_against).first
+    end
+    
+    # Describe instances
+    def describe_instances(o={})
+      instants = Ec2Response.describe_instances(ec2.describe_instances).select_with_hash(o)
+      return [] if instants.empty?
+      ec2_instances = instants.collect{|i| Ec2Instance.new(dsl_options.merge(i))}
+      ec2_instances.sort {|a,b| a[:launch_time].to_i <=> b[:launch_time].to_i }
+    end
+    
+    # Terminate an instance (or instances) by passing :instance_id and :instance_ids
+    def terminate_instance!(o={})
+      raise StandardError.new("You must pass an instance_id when terminating an instance with ec2") unless o[:instance_id] || o[:instance_ids]
+      instance_ids = o[:instance_ids] || [o[:instance_id]]
+      response = ec2.terminate_instances(instance_ids)
+      response.collect{|i| Ec2Instance.new(Ec2Response.pp_format(i)) }
+    end
+    
+    
+=begin rdoc
+  Helper methods for the Ec2 Cloud Provider. Helpers are not necessarily supported across all CloudProviders
+=end
+    # Are we running on amazon?
+    def amazon?
+      !['https://ec2.amazonaws.com', 
+       'https://us-east-1.ec2.amazonaws.com', 
+       'https://eu-west-1.ec2.amazonaws.com'
+       ].include?(ec2_url)
+    end
+    
+    # Read  yaml file and use it to set environment variables and local variables.
+    def set_aws_env_from_yml_file(filename='/etc/poolparty/env.yml')
+      aws = self.class.load_keys_from_file(filename)
+      aws.each{|k,v| ENV[k.upcase]=v.to_s}
+      set_vars_from_options aws
+    end
+    
+    # Save aws keys and env variables to a yaml file
+    def save_aws_env_to_yml(filename='/etc/poolparty/aws.yml')
+      File.open(filename, 'w') {|f| f<<YAML::dump(aws_hash) }
+    end
+    
+    # Return a hash of the aws keys and environment variables
+    # If base_dir string is provided as second argument, replace path to 
+    # file based variables, such as cert, with the base_dir.
+    def aws_hash(opts={}, base_dir=nil)
+      aws={
+        :user_id            => user_id,
+        :private_key        => private_key,
+        :cert               => cert,
+        :access_key         => access_key,
+        :secret_access_key  => secret_access_key,
+        :ec2_url            => ec2_url,
+        :s3_url             => s3_url,
+        :cloud_cert    => cloud_cert
+      }.merge(opts)
+      if base_dir
+        aws[:cert] = "#{base_dir}/#{File.basename(cert)}" if cert
+        aws[:private_key] = "#{base_dir}/#{File.basename(private_key)}" if private_key
+        aws[:cloud_cert] = "#{base_dir}/#{File.basename(cloud_cert)}" if cloud_cert
+      end
+      aws.reject{|k,v| v.nil?}
+    end
+    
+    # shortcut to 
+    # ec2-add-keypair name > ~./.ec2/kname
+    def create_keypair(kname, path='~/.ec2')
+      ` ec2-add-keypair #{kname} > #{path}/#{kname} &&  chmod 600 #{path}/#{kname}`
+    end
+    
+  end
+end

data/lib/cloud_providers/ec2/ec2_helpers.rb

@@ -0,0 +1,45 @@
+module CloudProviders
+  module Ec2Helpers
+    
+    # Associate an address with the instance using ec2
+    # Get the next_unused_elastic_ip
+    # and if there is one, associate the instance to the 
+    # public ip
+    def associate_address()
+      raise StandardError.new('Not Implemented Yet')
+    end
+    
+    # Get the next usable elastic ip
+    # First, get the list of addresses from ec2 that the client
+    # has access to, then select only the ones that are not associated
+    # with an instance.
+    # If the cloud has set elastic_ips to use, then, using the 
+    # intersection of the unused ips and those, find the first one available
+    # and return that.
+    def next_unused_elastic_ip
+      raise StandardError.new('Not Implemented Yet')
+      if elastic_ips.empty?
+        nil
+      else
+      end
+    end
+
+    # Help create a keypair for the cloud
+    # This is a helper to create the keypair and add them to the cloud for you
+    # def create_keypair
+    #   return false unless keypair
+    #   unless ::File.exists?( new_keypair_path )
+    #     FileUtils.mkdir_p ::File.dirname( new_keypair_path )
+    #     vputs "Creating keypair: #{keypair} in #{new_keypair_path}"
+    #     Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+    #   end
+    # end
+    
+    # wrapper for remote base to perform a snapshot backup for the ebs volume
+    # def create_snapshot
+    #   return nil if ebs_volume_id.nil?
+    #   ec2.create_snapshot(:volume_id => ebs_volume_id)
+    # end
+    
+  end
+end