auser-poolparty 1.2.12 → 1.3.0

data/lib/cloud_providers/ec2/ec2_instance.rb

@@ -0,0 +1,120 @@
+require "#{::File.dirname(__FILE__)}/ec2"
+
+module CloudProviders
+
+  class Ec2Instance < CloudProviderInstance
+    
+    default_options(
+      Ec2.default_options.merge({
+        :launching_time   => nil,
+        :private_dns_name => nil,
+        :kernel_id        => nil,
+        :ramdisk_id       => nil,
+        :launch_time      => nil,
+        :instance_id      => nil,
+        :launch_index     => nil,
+        :public_ip        => nil,
+        :internal_ip      => nil,
+        }
+      )
+    )
+    
+    # A new instance will be created from a hash.
+    # The parent clouds describe_instances list will be searched for the first one matching any of this instance's provided unique identifiers.
+    # If an instance is found, this instance's properties will be set to the properties provided
+    # If the found instance has properties of the same key as the provided options, the found instance's values will override the passed in options
+    def initialize(opts={}, &block)
+      opts.delete(:id)  # metavirt (in case your using it) passes an id that we do not want to set
+      set_vars_from_options(opts)
+      super
+    end
+    
+    # Printing. This is how we extract the instances into the listing on the 
+    # local side into the local listing file
+    def to_s
+      "#{name}\t#{dns_name}\t#{instance_id}"
+    end
+    
+    # #TODO: test or remove
+    # def hosts_file_listing_for(cl)
+    #   string = (cl.name == cloud.name) ? "#{name}.#{my_cloud.name}\t#{name}" : "#{name}.#{my_cloud.name}"
+    #   "#{internal_ip}\t#{string}"
+    # end
+    
+    def cloud_provider(o={}, &block)
+      @cloud_provider ||= if cloud
+        cloud.cloud_provider
+      else
+        options_for_cloud_provider = o.choose{|k,v| Ec2.default_options.has_key?(k)}
+        Ec2.new( options_for_cloud_provider, &block)
+      end
+    end
+    
+    # Access the right_aws instance directly
+    def right_aws
+      cloud_provider.ec2.describe_instances instance_id
+    end
+    
+    # add ec2 specific configuration steps
+    def configure!(opts={})
+      cloud opts[:cloud] if opts[:cloud]
+      raise StandardError.new("cloud is not defined.  It must be defined to run configure on the instance") unless cloud
+      vputs "configuring ec2 instance #{instance_id}."
+      ec2_dir = "/etc/poolparty/ec2"
+      FileUtils.mkdir_p(cloud.tmp_path/ec2_dir) unless File.directory?(cloud.tmp_path/ec2_dir)
+      run ["mkdir -p #{ec2_dir}"]
+      # Save a yaml file of aws varibles and send to the instance
+      File.open(cloud.tmp_path/ec2_dir/'aws.yml', 'w') do |f|
+        f<<YAML::dump(cloud_provider.aws_hash(ec2_dir))  #TODO: don't save sensitive info in /tmp
+      end
+      # We scp these files directly to the instance so to reduce the risk of accidentally leaving them in an insecure location
+      scp(:source=>cert,        :destination=>ec2_dir/File.basename(cert)) if cert
+      scp(:source=>private_key, :destination=>ec2_dir/File.basename(private_key)) if private_key
+      scp(:source=>cloud_cert,  :destination=>ec2_dir/File.basename(cloud_cert)) if cloud_cert
+      # TODO: install_ec2_tools
+      super opts
+      vputs "completed configuring instance  #{instance_id}."
+    end
+    
+    def rsync_excludes(array_of_abs_paths_to_exclude=nil)
+      array_of_abs_paths_to_exclude ||= %w( /sys
+                             /proc
+                             /dev/pts
+                             /dev
+                             /media
+                             /mnt
+                             /proc
+                             /sys
+                             /etc/udev/rules.d/70-persistent-net.rules
+                             /etc/udev/rules.d/z25_persistent-net.rules
+                            )
+      array_of_abs_paths_to_exclude.inject(''){|str, path| str<<"--exclude=#{path}"; str}
+    end
+    
+    # create an image file and copy this instance to the image file.
+    def make_image(opts={})
+      opts = {:volume       => '/',
+              :size         => 6000,
+              :destination  => '/mnt/bundle',
+              :prefix       => image_id,
+              :cert         => cert,
+              :exclude      => nil,
+              :kernel       => kernel_id,
+              :ramdisk      => ramdisk_id,
+              :ec2cert      => cloud_cert
+              }.merge(opts)
+      image_file = File.join(opts[:destination], opts[:prefix] )
+      cmds = ["mkdir -p #{opts[:destination]}"]
+      cmds << "dd if=/dev/zero of=#{image_file} bs=1M count=#{opts[:size]}"
+      cmds << "mkfs.ext3 -F -j  #{image_file}"
+      cmds << "mkdir -p #{opts[:destination]}/loop"
+      cmds << "mount -o loop #{image_file} #{opts[:destination]}/loop"
+      cmds << "rsync -ax #{rsync_excludes(opts[:exclude])} #{opts[:volume]}/ #{opts[:destination]}/loop/"
+      cmds << "umount #{opts[:destination]}/loop"
+      self.run cmds
+      image_file
+    end
+    
+  end
+
+end

data/lib/cloud_providers/ec2/ec2_response.rb

@@ -0,0 +1,79 @@
+module CloudProviders
+  
+  # Convert the standard right_aws gem's response object to the poolparty format
+  class Ec2Response
+    
+    # The right_aws gem returns a response similar to the following:
+    
+    # [{:aws_product_codes=>[],
+    #   :dns_name=>"192.168.4.198",
+    #   :aws_state_code=>"16",
+    #   :private_dns_name=>"192.168.4.198",
+    #   :aws_reason=>"",
+    #   :aws_instance_type=>"m1.large",
+    #   :aws_owner=>"admin",
+    #   :ami_launch_index=>"0",
+    #   :aws_launch_time=>"2009-07-11T03:21:02.113Z",
+    #   :aws_reservation_id=>"r-3753070E",
+    #   :aws_kernel_id=>"eki-AEAC17DE",
+    #   :ssh_key_name=>"sample_keypair",
+    #   :aws_state=>"running",
+    #   :aws_groups=>["default"],
+    #   :aws_ramdisk_id=>"eri-17051928",
+    #   :aws_instance_id=>"i-3E90078F",
+    #   :aws_availability_zone=>"jordan",
+    #   :aws_image_id=>"emi-39921602"}]
+    
+    # Convert the standard reponse into output similar to this example
+    
+    # {:dns_name          =>"ec2-75-101-175-49.compute-1.amazonaws.com",
+    #  :private_dns_name  =>"domU-11-31-39-00-DC-78.compute-1.internal",
+    #  :reason            =>nil,
+    #  :instance_state    =>{"name"=>"running", "code"=>"16"},
+    #  :kernel_id         =>"aki-a71cf9ce",
+    #  :ramdisk_id        =>"ari-a51cf9cc",
+    #  :product_codes     =>nil,
+    #  :image_id          =>"ami-bf5eb9d6",
+    #  :launch_time       =>"2009-05-29T05:07:09.000Z",
+    #  :key_name          =>"poolname_cloudname",
+    #  :instance_id       =>"i-1b7b2942",
+    #  :launch_index      =>"0",
+    #  :instance_type     =>"m1.small"}
+    
+    # Selects the first instance if an index is not given.
+    def self.describe_instance(response_array, index=0)
+      pp_format(response_array[index])
+    end
+    
+    # Convert the standard response hash to format used throughout the rest of PoolParty code.
+    def self.describe_instances(response)
+      response.collect{|inst| pp_format(inst) }
+    end
+    
+    def self.pp_format(response_hash)
+      munged = {}
+      response_hash.each{|k,v| munged[k.to_s.gsub('aws_', '').to_sym] = v}
+      munged[:internal_ip]      = convert_from_ec2_dns_to_ip(munged[:private_dns_name]) if munged[:private_dns_name]
+      munged[:public_ip]        = convert_from_ec2_dns_to_ip(munged[:dns_name]) if munged[:dns_name]
+      munged[:launch_time]      = parse_datetime(munged[:launch_time]) if munged[:launch_time]
+      munged[:ami_launch_index] = munged[:launch_index] if munged[:launch_index]  #TODO: deprecate ami_launch_index
+      munged[:keypair_name]     = munged.delete(:ssh_key_name)
+      munged[:status]           = munged.delete(:state) || munged[:shutdown_state]
+      munged
+    end
+    
+    # parse IP address from aws dns name
+    def self.convert_from_ec2_dns_to_ip(str=nil)
+      return str if str.nil?
+      # if using eucalyptus we may have the raw IP already, if so just return it
+      return str if str.match( /^\d{1,3}\.\d{1,3}.\d{1,3}\.\d{1,3}$/ )
+      str.scan(/-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})/).flatten.join('.')
+    end
+    
+    def self.parse_datetime(str)
+      Time.parse( str.chomp )
+    end
+    
+  end
+  
+end

data/lib/cloud_providers/ssh/ssh.rb

@@ -0,0 +1,68 @@
+=begin rdoc
+  This is an example CloudProvider to be used as a template for implementing new CloudProviders
+=end
+require "#{File.dirname(__FILE__)}/ssh_instance"
+
+module CloudProviders
+  class Ssh < CloudProvider
+    default_options(
+      CloudProvider.default_options.merge(
+        :hosts  => [] # list of hostnames, or addresses
+      )
+    )
+    
+    # By default we consider all hosts to be running
+    # available_hosts represents those that we are not "running" right now
+    def available_hosts
+      @available_hosts ||= []
+    end
+    
+    def running_hosts
+      hosts - available_hosts
+    end
+    
+    # Select just the instance options that are relevant to the SshInstance
+    def instance_options(opts)
+      i_opts = dsl_options.merge(opts)
+      # i_opts = SshInstance.default_options.merge(i_opts)
+      # i_opts.choose{|k,v| SshInstance.default_options.keys.include?(k)}
+    end
+    
+    # Launch a new instance
+    def run_instance(o={})
+      raise StandardError.new("You have no available hosts") if available_hosts.empty?
+      host = available_hosts.first
+      new_host = SshInstance.new(instance_options(o.merge(:name=>host)))
+      new_host.keypair
+      if new_host && new_host.refresh!
+        available_hosts.delete host
+        new_host
+      else
+        raise StandardError.new("Unable to connect to host #{host}")
+      end
+    end
+    
+    # Terminate an instance by name, or the last instance
+    def terminate_instance!(o={})
+      thost = o[:name] || running_hosts.last
+      available_hosts << thost.clone
+      SshInstance.new(instance_options(o.merge(:name=>thost, :status=>'terminated')))
+    end
+
+    # Returns an instance object for the host.  You must pass the :name option
+    def describe_instance(opts={:name=>nil})
+      raise StandardError.new("you must provide a name") unless opts[:name]
+      SshInstance.new(instance_options(o.merge(:name=>opts[:name])))
+    end
+    
+    def describe_instances(o={})
+      hosts.collect{|n| SshInstance.new(instance_options(o.merge(:name=>n)))}
+    end
+    
+    def nodes(hsh={})
+      results = running_hosts.collect{|n| SshInstance.new(instance_options(:name=>n))}
+      results.select_with_hash(hsh)
+    end
+    
+  end
+end

data/lib/cloud_providers/ssh/ssh_instance.rb

@@ -0,0 +1,41 @@
+module CloudProviders
+  
+  class SshInstance < CloudProviderInstance
+      
+      default_options(
+        :user     => 'root',
+        :status   => 'running',
+        :hostname => nil,
+        :cloud    => nil
+      )
+      
+      def instance_id
+        self.name
+      end
+      
+      def initialize(opts={}, &block)
+        @host = opts[:name] || name || dns_name
+        super
+      end
+      
+      def cloud_provider(o={}, &block)
+        @cloud_provider ||= if cloud
+          cloud.cloud_provider
+        else
+          options_for_cloud_provider = o.choose{|k,v| Ssh.default_options.has_key?(k)}
+          Ssh.new( options_for_cloud_provider, &block)
+        end
+      end
+      
+      def refresh!
+        run 'uptime'
+        name(run('hostname'))
+        dns_name(name)
+        status('running') if name
+        # :internal_ip  => nil, # TODO
+        # :public_ip    => nil, #TODO
+        self
+      end
+    
+    end
+end

data/lib/cloud_providers/vmware/vmware.rb

@@ -0,0 +1,127 @@
+=begin rdoc
+  The Vmrun remote base uses the vmrun command to implement a cloud remoter base.
+  The Vmrun remoter base has been tested with Vmware fusion on the mac.
+  
+  In order to use the Vmrun remoter base you will need to setup a few things.
+  First, you will need to have the Vmware fusion installed and vmrun command in your path.
+  The default location of the vmrun binary is /Library/Application Support/VMware Fusion/vmrun.
+  You will then of course need a virtual machine installed and available.  
+  Once you have your instance installed and running, you need to setup your ssh keys so that poolparty can ssh into the instance. Get the ip of you instnace with ifconfig inside the running instance.
+  For example:
+    
+    ssh root@172.0.1.129 "mkdir /root/.ssh && chmod 600 /root/.ssh"
+    scp my_key.pub root@172.0.1.129:/root/.ssh/authorized_keys
+    
+  provide a using :vmrun block in your clouds.rb
+  for example:
+    
+    using :vmrun do
+      vmx_hash(::File.expand_path("~/Documents/Virtual\ Machines.localized/Ubuntu-jaunty.vmwarevm/Ubuntu-jaunty.vmx") => '172.16.68.129')
+    end
+   
+   The vmx file fulfills a similar purpose as the ami id in ec2. Note that expand path.  vmrun return fuill paths, so we must provide full paths so things match up later. W
+   Vmrun does not provide a meta server like ec2 has, so you need to list the ip address of your VM.  For this reason it is recommended that you use NAT addressing on your VM to maintain consistent addressing across different physical networks.
+  
+  Also, note that vmrun does not copy the VM to a new distinct VM on each run, so if you want to be able to start from a known state, you should make a snapshot before using your vm with poolparty.  Then, you can rollback to this initial state if you want to ensure you can repeat a fresh cloud-start.
+
+=end
+require "open3"
+
+require "#{File.dirname(__FILE__)}/vmware_instance"
+
+module CloudProviders
+  class Vmware < CloudProvider
+    include SearchablePaths
+    
+    has_searchable_paths(:paths => [
+      File.expand_path("/Library/Application Support/VMware Fusion"),
+      File.expand_path("#{ENV["HOME"]}/Library/Application Support/VMware Fusion"),
+      "/usr/bin"
+    ])
+    
+    default_options(
+      :image_id             => nil,
+      :public_ip            => nil,
+      :default_cli_options  => 'gui',
+      :terminate_options    => 'soft',
+      :keypair_name         => nil,
+      :images_repo_path     => ::File.expand_path("#{ENV["HOME"]}/Documents/Virtual_Machines.localized/")
+      )
+    
+    attr_reader :cloud
+    
+    def initialize(o={}, &block)
+      @cloud = o.delete(:cloud)
+      super
+    end
+    
+    # Start a new instance with the given options
+    def run_instance(o={})
+      vmrun("start #{vmx_file}")
+      vmware_instance
+    end
+    
+    # Will select the first instance matching the provided criteria hash
+    def describe_instance(hash_of_criteria_to_select_instance_against)
+      describe_instances.first
+    end
+    
+    # Describe instances
+    def describe_instances(o={})
+      output = vmrun("list")
+      if output.first =~ /Total running VMs: 0/
+        []
+      else
+        output.shift if output.first =~ /Total running/
+        output.map do |line|
+          vmware_instance(:vmx_file => line)
+        end
+      end
+    end
+    
+    # Terminate an instance (or instances) by passing :instance_id and :instance_ids
+    def terminate_instance!(o={})
+      set_vars_from_options o
+      vmrun("stop #{vmx_file}")
+      describe_instances
+    end
+    
+    # The nodes in the cloud_provider
+    def nodes(hsh={})
+      describe_instances
+    end
+    
+    # Basic vmware_instance
+    def vmware_instance(o={})
+      VmwareInstance.new( :instance_id => (o[:vmx_file] || vmx_file), :public_ip => public_ip, :dns_name => public_ip,
+                          :cloud_provider => self.dsl_options, :keypair_name => keypair.basename)
+    end
+    
+    # Search for the vmrun binary
+    def path_to_binary
+      @path_to_binary ||= search_in_known_locations("vmrun")
+    end
+    
+    # Run vmrun on the command-line
+    def vmrun(cmd, o={})
+      stdin, stdout, stderr = Open3.popen3("'#{path_to_binary}' #{cmd}")
+      unless $?.success?
+        $stderr.puts "FAILED: #{cmd}\n code = #{$?}"
+        raise StandardError.new("ERROR: vmrun") if o.delete(:raise_on_error)
+      end
+      stdout.readlines
+    end
+        
+    # Search for the vmx_file
+    def vmx_file
+      return @vmx_file if @vmx_file
+      o = if File.file?(f = File.expand_path(image_id))
+        f
+      elsif File.file?(f = File.expand_path(images_repo_path, image_id))
+        f
+      end
+      @vmx_file = "'#{o}'"
+    end
+    
+  end
+end

data/lib/cloud_providers/vmware/vmware_instance.rb

@@ -0,0 +1,56 @@
+require "#{::File.dirname(__FILE__)}/vmware"
+
+module CloudProviders
+
+  class VmwareInstance < CloudProviderInstance
+    
+    default_options(
+      :launching_time   => nil,
+      :vmx_file         => nil,
+      :instance_id      => nil,
+      :public_ip        => nil,
+      :status           => "running",
+      :dns_name         => nil
+    )
+    
+    # A new instance will be created from a hash.
+    # The parent clouds describe_instances list will be searched for the first one matching any of this instance's provided unique identifiers.
+    # If an instance is found, this instance's properties will be set to the properties provided
+    # If the found instance has properties of the same key as the provided options, the found instance's values will override the passed in options
+    def initialize(opts={}, &block)
+      set_vars_from_options(opts)
+      super
+    end
+    
+    # Printing. This is how we extract the instances into the listing on the 
+    # local side into the local listing file
+    def to_s
+      "#{name}\t#{dns_name}\t#{instance_id}"
+    end
+    
+    def status
+      "running"
+    end
+    
+    # Since we aren't actually getting any ips from a list, only using 
+    # the one vmx file, we don't actually need to refresh anything
+    def wait_for_public_ip(timeout=60)
+      public_ip      
+    end
+    
+    def refresh!
+      self
+    end
+    
+    def cloud_provider(o={}, &block)
+      @cloud_provider ||= if cloud
+        cloud.cloud_provider
+      else        
+        options_for_cloud_provider = o.choose{|k,v| Vmware.default_options.has_key?(k)}
+        Vmware.new( options_for_cloud_provider, &block)
+      end
+    end
+    
+  end
+
+end

data/lib/cloud_providers.rb

@@ -0,0 +1,19 @@
+=begin rdoc
+  CloudProvider is the base class for cloud computing services such as Ec2, Eucalyptus - where your servers run.
+=end
+module CloudProviders
+
+  # List of all defined cloud_providers
+  def self.all
+    @all ||= []
+  end
+
+end
+
+require File.dirname(__FILE__)+"/cloud_providers/connections"
+require File.dirname(__FILE__)+"/cloud_providers/cloud_provider"
+require File.dirname(__FILE__)+"/cloud_providers/cloud_provider_instance"
+
+%w(ec2 vmware ssh).each do |lib|
+  require "cloud_providers/#{lib}/#{lib}"
+end

data/lib/core/array.rb

@@ -0,0 +1,54 @@
+=begin rdoc
+  Array extensions
+=end
+
+require "enumerator"
+class Array
+  def collect_with_index &block
+    self.enum_for(:each_with_index).collect &block
+  end
+
+  # Example  nodes.select_with_hash(:status=>'running')
+  def select_with_hash(conditions={})
+    return self if conditions.empty?
+    select do |node|
+      conditions.any? do |k,v|
+        ( node.has_key?(k) && node[k]==v ) or ( node.respond_to?(k) && node.send(k)==v )
+      end
+    end
+  end
+  
+  def wrapping_next(id)
+    raise "Element #{id} not in array" unless index(id)
+    index(id) >= size-1 ? at(0) : at(index(id)+1)
+  end
+  
+  # Swap elements of an array
+  def swap!(a,b)
+    self[a], self[b] = self[b], self[a]
+    self
+  end
+  
+  # Rotate an array
+  # Example:
+  #   arr = %w(a b c d e f)
+  #   arr2 = arr.clone.rotate
+  # 
+  #   arr.zip(arr2) => 
+  #   [["a", "b"], ["b", "c"], ["c", "d"], ["d", "e"], ["e", "f"], ["f", "a"]]  
+  def rotate
+    push shift
+  end
+  
+  def zip_offset(offset=0)
+    arr2 = self.dup
+    
+    out = []
+    self.each_with_index do |ele,i|
+      break if i == (self.size - offset)
+      out[i] = [ele, arr2[i+1]]
+    end
+    out
+  end
+    
+end

data/lib/core/hash.rb

@@ -0,0 +1,80 @@
+=begin rdoc
+  Hash extentions
+=end
+class Hash
+    
+  # Return a hash of all the elements where the block evaluates to true
+  def choose(&block)
+    Hash[*self.select(&block).inject([]){|res,(k,v)| res << k << v}]
+  end
+
+  def to_instance_variables(inst=nil)
+    each do |k,v|
+      inst.instance_variable_set "@#{k}", v
+      inst.class.send :attr_reader, k if inst
+    end
+  end
+  
+  # extracted from activesupport
+  # Returns an array of the values at the specified indices:
+  #
+  #   hash = HashWithIndifferentAccess.new
+  #   hash[:a] = "x"
+  #   hash[:b] = "y"
+  #   hash.values_at("a", "b") # => ["x", "y"]
+  def values_at(*indices)
+    indices.collect {|key| self[key]}.compact
+  end
+  
+  def next_sorted_key(from)
+    idx = (size - keys.sort.index(from))
+    keys.sort[idx - 1]
+  end
+  
+  def stringify_keys
+    dup.stringify_keys!
+  end
+  
+  # Converts all of the keys to strings
+  def stringify_keys!
+    keys.each{|k| 
+      v = delete(k)
+      self[k.to_s] = v
+      v.stringify_keys! if v.is_a?(Hash)
+      v.each{|p| p.stringify_keys! if p.is_a?(Hash)} if v.is_a?(Array)
+    }
+    self
+  end
+  
+  def symbolize_keys(key_modifier=nil)
+    dup.symbolize_keys!(key_modifier)
+  end
+  
+  # Converts all of the keys to strings
+  # can pass in a :key_modifier that will be sent to each key, before being symbolized.
+  # This can be usefull if you want to downcase, or snake_case each key.
+  # >> {'Placement' => {'AvailabilityZone'=>"us-east-1a"} }.symbolize_keys(:snake_case)
+  # => {:placement=>{:availability_zone=>"us-east-1a"}}  
+  def symbolize_keys!(key_modifier=nil) 
+    keys.each{|k| 
+      v = delete(k)
+      if key_modifier && k.respond_to?(key_modifier)
+        k = k.send(key_modifier)
+      end
+      self[k.to_sym] = v
+      v.symbolize_keys!(key_modifier) if v.is_a?(Hash)
+      v.each{|p| p.symbolize_keys!(key_modifier) if p.is_a?(Hash)} if v.is_a?(Array)
+    }
+    self
+  end
+    
+  def method_missing(sym, *args, &block)
+    if has_key?(sym.to_sym)
+      fetch(sym)
+    elsif has_key?(sym.to_s)
+      fetch(sym.to_s)
+    else
+      super
+    end
+  end  
+end

data/lib/core/integer.rb

@@ -0,0 +1,11 @@
+class Integer
+  def months
+    self * 30.days
+  end
+  alias :month :months
+
+  def years
+    self * 365.25.days
+  end
+  alias :year :years
+end

data/lib/core/module.rb

@@ -0,0 +1,14 @@
+class Module
+  
+  # Default accessors
+  # Provides an accessor with a default class
+  # Usage:
+  #   default_attr_reader :a, ["b"]
+  def default_attr_reader(sym, default)
+    define_method :"#{sym}_default" do
+      default
+    end
+    module_eval "def #{sym};@#{sym} ||= #{sym}_default;end"
+  end
+  
+end