poolparty 1.3.15 → 1.4.0

data/examples/thrift/erlang/src/poolparty_types.erl

@@ -1,30 +0,0 @@
-%%
-%% Autogenerated by Thrift
-%%
-%% DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
-%%
-
--module(poolparty_types).
-
--include("poolparty_types.hrl").
-
--export([struct_info/1]).
-%% struct cloudQuery
-
-% -record(cloudQuery, {name}).
-
-struct_info('cloudQuery') ->
-  {struct, [{1, string}]}
-;
-
-%% struct cloudResponse
-
-% -record(cloudResponse, {name, command, response}).
-
-struct_info('cloudResponse') ->
-  {struct, [{1, string},
-  {2, string},
-  {3, {list, string}}]}
-;
-
-struct_info('i am a dummy struct') -> undefined.

data/examples/thrift/erlang/src/thrift_example.erl

@@ -1,32 +0,0 @@
-%%====================================================================
-%% thrift_example 
-%%  TODO: COMPLETE
-%% RUN:
-%%  make deps
-%%  make 
-%%  make run
-%%====================================================================
-
--module (thrift_example).
--include ("poolparty_types.hrl").
--include ("commandInterface_thrift.hrl").
--include ("poolparty_constants.hrl").
-
--compile (export_all).
-
-run() ->
-  {ok, P} = thrift_client:start_link("localhost", 11223, commandInterface_thrift),
-  O1 = cloud_query(P, "app", "name"),
-  io:format("Name: ~p~n", [O1]),
-  O2 = cloud_query(P, "app", "maximum_instances"),
-  io:format("maximum_instances: ~p~n", [O2]),
-  O3 = cloud_query(P, "app", "take_args", ["a", "b"]),
-  io:format("cloud_provider: ~p~n", [O3]).
-
-cloud_query(P, Name, Meth) ->
-  cloud_query(P, Name, Meth, "").
-  
-cloud_query(P, Name, Meth, Args) ->
-  Query = #cloudQuery{name=Name},
-  io:format("Query: ~p, ~p, ~p~n", [Query, Meth, Args]),
-  thrift_client:call(P, run_command, [Query, Meth, Args]).

data/examples/thrift/thrift_example.py

@@ -1,28 +0,0 @@
-import sys,os
-gen_py_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', '..', 'lib', 'proto', 'gen-py')
-sys.path.append(gen_py_path)
-
-from cloudthrift import CommandInterface
-from cloudthrift.constants import *
-
-from thrift import Thrift
-from thrift.transport import TSocket
-from thrift.transport import TTransport
-from thrift.protocol import TBinaryProtocol
-
-transport = TSocket.TSocket('localhost', 11223)
-transport = TTransport.TBufferedTransport(transport)
-protocol = TBinaryProtocol.TBinaryProtocol(transport)
-
-ci = CommandInterface.Client(protocol)
-
-transport.open()
-
-cloudquery = CloudQuery()
-cloudquery.name = "app"
-
-resp = ci.run_command(cloudquery, "name", "")
-print resp.response
-
-resp = ci.run_command(cloudquery, "maximum_instances", "")
-print resp.response

data/examples/thrift/thrift_example.rb

@@ -1,40 +0,0 @@
-#!/usr/bin/env ruby
-
-$:.unshift(File.dirname(__FILE__) + "/../../lib")
-$:.unshift(File.dirname(__FILE__) + "/../../examples")
-require "poolparty"
-# require "simple"
-
-$:.unshift("#{File.dirname(__FILE__)}/../../lib/proto/gen-rb")
-
-require "thrift"
-require "command_interface"
-require "poolparty_constants"
-require "poolparty_types"
-
-port = ARGV.pop || 11223
-
-transport = Thrift::BufferedTransport.new(Thrift::Socket.new('localhost', port))
-# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('192.168.2.31', port))
-# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('vm', port))
-protocol = Thrift::BinaryProtocol.new(transport)
-
-client = CloudThrift::CommandInterface::Client.new(protocol)
-transport.open()
-
-cld = CloudThrift::CloudQuery.new
-cld.name = 'pp2'
-# cld.name = 'monitored_app'
-# cld.name = 'vmware'
-
-resp = client.run_command(cld, "name", [])
-puts resp.response
-
-resp = client.run_command(cld, "maximum_instances", [])
-puts resp.response
-
-resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.91"])
-p resp.response
-
-resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.01"])
-p resp.response

data/examples/vmware.rb

@@ -1,28 +0,0 @@
-require "poolparty"
-require 'poolparty-extensions'
-
-pool "poolparty" do
-  instances 1
-  
-  cloud "vmware" do
-    keypair "id_rsa"
-    using :vmware do
-      image_id  "/Users/nmurray/Documents/VMware/Ubuntu-jaunty.vmwarevm/Ubuntu-jaunty.vmx"
-      public_ip "192.168.133.128"
-    end
-    has_hermes
-    has_file "/etc/motd", :content => "welcome to your instance"
-
-    has_convenience_helpers
-  end
-
-end
-
-# hermes
-# want to be able to run cloud configure to update it
-# but want to be able to pull from locally to configure it
-# submodules?  
-# node name
-# install path
-# how to upload the escript target system
-# how to upload the tar.gz of the release

data/lib/cloud_providers/cloud_provider_instance.rb

@@ -1,288 +0,0 @@
-require 'socket'
-require 'timeout'
-
-module CloudProviders
-  
-  class CloudProviderInstance
-      include Dslify
-      # include Enumerable
-      include Connections
-      include Callbacks
-      
-      default_options(
-        :cloud        => nil, # The cloud this instance belongs to, set automaticly if node is created thru cloud expansion
-        :name         => nil, # Name of the remote instance (internal usage)
-        :internal_ip  => nil, # Internal ip of the remote instance
-        :public_ip    => nil,
-        :dns_name     => nil,
-        :status       => nil, # Status of the remote instance
-        :launch_time  => nil,
-        :keypair_name => nil,
-        :cloud_name   => nil
-      )
-      
-      def initialize(opts={}, &block)
-        self.cloud_name= opts[:cloud].name if opts[:cloud]
-        set_vars_from_options(opts)
-        instance_eval(&block) if block
-        loaded
-      end
-      
-      # Returns an instance of Keypair
-      # You can pass either a filename which will be searched for in ~/.ec2/ and ~/.ssh/
-      # or you can pass a full filepath
-      def keypair(n=keypair_name)
-        cloud_provider.keypair(n)
-      end
-      
-      def cloud_provider(opts={}, &block)
-        raise StandardError.new("cloud_provider has not been implemented for this CloudProviderInstance ")
-      end
-      
-      # CLOUD PROVIDER METHODS
-      
-      # Bootstrap self.  Bootstrap runs as root, even if user is set
-      def bootstrap!(opts={})
-        callback :before_bootstrap
-         if !bootstrapped? || opts[:force]
-           old_user = user
-           @user = "root"
-           opts[:os] ||= os || determine_os
-           if cloud && cloud.bootstrap_script
-             opts[:filename] = cloud.bootstrap_script
-           end
-          script_file = Provision::Bootstrapper.bootstrap_script(opts)
-          scp(:source => script_file, :destination => "/tmp")
-          output = run("chmod +x /tmp/determine_os.sh; /bin/sh /tmp/#{File.basename(script_file)}")
-          @user = old_user
-          output.chomp if output
-        end
-        callback :after_bootstrap
-      end
-      
-      # Configure the node
-      def configure!(opts={})
-        ddputs("Configuring: #{self.name}")
-        bootstrap! unless bootstrapped?
-        callback :before_configure
-        raise StandardError.new("You must pass in a cloud to configure an instance") unless cloud
-        cloud.compile(self)        
-        
-        # scp(:source       => keypair.full_filepath, 
-        #     :destination  => "/etc/poolparty/keys/#{keypair.basename}")
-        
-        FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty" unless File.directory?(cloud.tmp_path/"etc"/"poolparty")
-        FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty"/"keys" unless File.directory?(cloud.tmp_path/"etc"/"poolparty"/"keys")
-        
-        FileUtils.cp keypair.full_filepath, cloud.tmp_path/"etc"/"poolparty"/"keys"/keypair.basename
-        File.open(cloud.tmp_path/"etc"/"poolparty"/"cloud_name", "w") {|f| f << cloud.name }
-        
-        pack_clouds_dot_rb_and_expected_directories
-        
-        dputs("Rsyncing #{cloud.tmp_path/"*"}")
-        rsync(:source => cloud.tmp_path/"*", :destination => "/")
-        run(cloud.dependency_resolver.compile_command)
-        callback :after_configure
-      end
-      
-      # Terminate self
-      def terminate!
-        cloud_provider.terminate_instance!(:instance_id => instance_id)
-      end
-      
-      # get the os, if it's not declared
-      def os(sym=nil)
-        if sym
-          dsl_options[:os] = sym
-        else
-          dsl_options[:os] ||= determine_os.to_sym
-        end
-      end
-      alias :platform :os  # Chef uses platform, aliased for conveneince
-      
-      def pack_clouds_dot_rb_and_expected_directories
-        %w(lib plugins).each do |dir|
-          if File.directory?(d = cloud.clouds_dot_rb_dir/dir)
-            dputs("Adding local path: #{d}")
-            FileUtils.cp_r d, cloud.tmp_path/cloud.base_config_directory, :verbose => true, :remove_destination => true # req'd for symlinks
-          end
-        end
-        FileUtils.cp cloud.clouds_dot_rb_file, cloud.tmp_path/"/etc/poolparty/clouds.rb"
-      end
-      # Determine the os
-      # Default to ubuntu
-      # Send the determine_os.sh script to the node and run it remotely
-      def determine_os
-        scp(:source => Provision::Bootstrapper.determine_os_script, :destination => "/tmp")
-        o = run("chmod +x /tmp/determine_os.sh; /bin/sh /tmp/determine_os.sh").chomp
-        o.empty? ? :ubuntu : o
-      end
-      
-      # Determine if the node is bootstrapped
-      def bootstrapped?
-        # @bootstrapped ||= !run('if [ -f /var/poolparty/bootstrapped ]; then echo "YES"; fi').match(/YES/).nil?
-        @bootstrapped ||= !run('if [ -f /var/poolparty/bootstrapped ]; then echo "YES"; fi').chomp.empty? || false
-      end
-      
-      # Wait for port
-      # Test if the port is open and catch failures in the connection
-      # Options
-      #   public_ip || default public_ip
-      #   retry_times || 5
-      def wait_for_port(port, opts={})
-        ip          = opts.delete(:public_ip)   || public_ip
-        retry_times = opts.delete(:retry_times) || 10
-        pause_time  = opts.delete(:pause_time)  || 1
-        
-        retry_times.times do |i| 
-          if is_port_open?(ip, port, opts)
-            return true
-          else
-            sleep pause_time
-          end
-        end
-        false
-      end
-      
-      # Wait for a public ip to be assigned, refreshing the instance data from the cloud provider on each query
-      # Default timeout value of 60 seconds, can be overriden by passing {:timeout=>seconds}
-      def wait_for_public_ip(timeout=60)
-        ddputs("Waiting for public ip")
-        begin
-          Timeout::timeout(timeout) do
-            loop do
-              self.refresh!
-              ddputs("After refreshing, the public_ip is: #{public_ip}")
-              return public_ip if valid_ip?(public_ip)
-              print '.'
-              sleep 2
-            end
-          end
-        rescue Timeout::Error
-          return false
-        end
-      end
-      
-      def terminate!
-        cloud_provider.terminate_instance!(:instance_id=>self.instance_id).first
-      end
-      
-      # Refresh the node with fresh data from the cloud provider.
-      # This is often usefully to update a recently launched instance, in case you want to trigger new behavior once the state changes ot 'running' and an ip is assigned
-      # Added rescue, but not sure if this is a proper fix yet. Will need to test on live instances to ensure
-      def refresh!
-        dsl_options = cloud_provider.describe_instance(:instance_id => self.instance_id).dsl_options rescue {}
-        self.dsl_options.merge!(dsl_options)
-        self
-      end
-      
-      ## hash like methods
-      def each
-        dsl_options.each{ |k,v| yield k,v }
-      end
-      
-      def [](k)
-        if dsl_options.has_key? k
-          dsl_options[k]
-        else
-          nil
-        end
-      end
-      
-      def []=(k,v)
-        dsl_options[k] = v
-      end
-      
-      def has_key?(key)
-        dsl_options.has_key?(key)
-      end
-      
-      def keys
-        dsl_options.keys
-      end
-         
-      def values
-        dsl_options.values
-      end
-      
-      def to_hash
-        dsl_options
-      end
-      ##end of hash like methods
-      
-      # Is this instance running?
-      def running?
-        !(status =~ /running/).nil?
-      end
-      # Is this instance pending?
-      def pending?
-        !(status =~ /pending/).nil?
-      end
-      # Is this instance terminating?
-      def terminating?
-        !(status =~ /shutting/).nil?
-      end
-      # Has this instance been terminated?
-      def terminated?
-        !(status =~ /terminated/).nil?
-      end
-      
-      def elapsed_runtime
-        Time.now.to_i - launching_time.to_time.to_i
-      end
-      
-      # Callbacks
-      def loaded
-      end
-            
-      def on_all_callbacks(call_time, *args, &block)
-        cloud.callback call_time if cloud
-        super
-      end
-      
-      def before_bootstrap
-      end
-      def after_bootstrap
-      end
-      def before_configure
-      end
-      def after_configure
-      end
-      
-      # The instances is only valid if there is an internal_ip and a name
-      def valid?
-        (internal_ip.nil? || name.nil?) ? false : true
-      end
-      
-      private
-      
-      def valid_ip?(ip)
-        ip && ip != '0.0.0.0' && ip != ''
-      end
-
-      # Test for open port by opening a socket
-      # on the ip and closing the socket
-      def is_port_open?(ip, port, opts={})
-        timeout = opts[:timeout] || 1
-        begin
-          Timeout::timeout(timeout) do
-            begin
-              s = TCPSocket.new(ip, port)
-              s.close
-              ddputs("Connected to #{ip}:#{port} - Port is open and good to go")
-              return true
-              puts ','
-            rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
-              ddputs("Port #{port} on #{ip} is not accessible (yet)")
-              return false
-            end
-          end
-        rescue Timeout::Error
-        end
-        ddputs("Port #{port} on #{ip} is not accessible")
-        return false
-      end
-      
-    end
-    
-end

data/lib/cloud_providers/ec2/ec2_helpers.rb

@@ -1,100 +0,0 @@
-module CloudProviders
-  module Ec2Helpers
-    
-    # VOLUMES    
-    def attach_volume(instance_id, volume_id=next_unused_volume, device="/dev/sdh")
-      ec2.attach_volume(volume_id, instance_id, device)
-    end
-    
-    def next_unused_volume
-      if all_volumes.empty?
-        nil
-      else
-        available_volumes.first
-      end
-    end    
-        
-    private
-    
-    def all_volumes
-      ebs_volumes.empty? ? [] : ec2.describe_volumes.select {|v| ebs_volumes.include?(v[:aws_id]) }
-    end
-    
-    def available_volumes
-      all_volumes.select {|v| v[:aws_status] == 'available' }
-    end
-    
-    def unavailable_volumes
-      all_volumes.reject {|v| available_volumes.include?(v) }
-    end
-    
-    public
-    
-    # SECURITY GROUPS
-    def security_groups(list=[])
-      ec2.describe_security_groups(list)
-    end
-    
-    public
-    
-    # ELASTIC IPS
-    
-    # Associate an address with the instance using ec2
-    # Get the next_unused_elastic_ip
-    # and if there is one, associate the instance to the 
-    # public ip
-    def associate_address(instance_id)
-      new_ip = next_unused_elastic_ip
-      ec2.associate_address(instance_id, new_ip)
-      loop do
-        if describe_instance(:instance_id => instance_id).public_ip == new_ip
-          return new_ip
-        end
-        sleep 1
-      end
-    end
-    
-    # Get the next usable elastic ip
-    # First, get the list of addresses from ec2 that the client
-    # has access to, then select only the ones that are not associated
-    # with an instance.
-    # If the cloud has set elastic_ips to use, then, using the 
-    # intersection of the unused ips and those, find the first one available
-    # and return that.
-    def next_unused_elastic_ip
-      if unusued_elastic_ips.empty?
-        nil
-      else
-        unusued_elastic_ips.first
-      end
-    end
-    
-    private
-    
-    def all_elastic_ips
-      elastic_ips.empty? ? [] : ec2.describe_addresses & elastic_ips
-    end
-    
-    def unusued_elastic_ips
-      all_elastic_ips.select {|i| i[:instance_id] == nil }
-    end
-    
-    # Help create a keypair for the cloud
-    # This is a helper to create the keypair and add them to the cloud for you
-    # def create_keypair
-    #   return false unless keypair
-    #   unless ::File.exists?( new_keypair_path )
-    #     FileUtils.mkdir_p ::File.dirname( new_keypair_path )
-    #     vputs "Creating keypair: #{keypair} in #{new_keypair_path}"
-    #     Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
-    #   end
-    # end
-    
-    # wrapper for remote base to perform a snapshot backup for the ebs volume
-    # def create_snapshot
-    #   return nil if ebs_volume_id.nil?
-    #   ec2.create_snapshot(:volume_id => ebs_volume_id)
-    # end
-    
-  end
-end

data/lib/cloud_providers/ec2/ec2_response.rb

@@ -1,79 +0,0 @@
-module CloudProviders
-  
-  # Convert the standard right_aws gem's response object to the poolparty format
-  class Ec2Response
-    
-    # The right_aws gem returns a response similar to the following:
-    
-    # [{:aws_product_codes=>[],
-    #   :dns_name=>"192.168.4.198",
-    #   :aws_state_code=>"16",
-    #   :private_dns_name=>"192.168.4.198",
-    #   :aws_reason=>"",
-    #   :aws_instance_type=>"m1.large",
-    #   :aws_owner=>"admin",
-    #   :ami_launch_index=>"0",
-    #   :aws_launch_time=>"2009-07-11T03:21:02.113Z",
-    #   :aws_reservation_id=>"r-3753070E",
-    #   :aws_kernel_id=>"eki-AEAC17DE",
-    #   :ssh_key_name=>"sample_keypair",
-    #   :aws_state=>"running",
-    #   :aws_groups=>["default"],
-    #   :aws_ramdisk_id=>"eri-17051928",
-    #   :aws_instance_id=>"i-3E90078F",
-    #   :aws_availability_zone=>"jordan",
-    #   :aws_image_id=>"emi-39921602"}]
-    
-    # Convert the standard reponse into output similar to this example
-    
-    # {:dns_name          =>"ec2-75-101-175-49.compute-1.amazonaws.com",
-    #  :private_dns_name  =>"domU-11-31-39-00-DC-78.compute-1.internal",
-    #  :reason            =>nil,
-    #  :instance_state    =>{"name"=>"running", "code"=>"16"},
-    #  :kernel_id         =>"aki-a71cf9ce",
-    #  :ramdisk_id        =>"ari-a51cf9cc",
-    #  :product_codes     =>nil,
-    #  :image_id          =>"ami-bf5eb9d6",
-    #  :launch_time       =>"2009-05-29T05:07:09.000Z",
-    #  :key_name          =>"poolname_cloudname",
-    #  :instance_id       =>"i-1b7b2942",
-    #  :launch_index      =>"0",
-    #  :instance_type     =>"m1.small"}
-    
-    # Selects the first instance if an index is not given.
-    def self.describe_instance(response_array, index=0)
-      pp_format(response_array[index])
-    end
-    
-    # Convert the standard response hash to format used throughout the rest of PoolParty code.
-    def self.describe_instances(response)
-      response.collect{|inst| pp_format(inst) }
-    end
-    
-    def self.pp_format(response_hash)
-      munged = {}
-      response_hash.each{|k,v| munged[k.to_s.gsub('aws_', '').to_sym] = v}
-      munged[:internal_ip]      = convert_from_ec2_dns_to_ip(munged[:private_dns_name]) if munged[:private_dns_name]
-      munged[:public_ip]        = convert_from_ec2_dns_to_ip(munged[:dns_name]) if munged[:dns_name]
-      munged[:launch_time]      = parse_datetime(munged[:launch_time]) if munged[:launch_time]
-      munged[:ami_launch_index] = munged[:launch_index] if munged[:launch_index]  #TODO: deprecate ami_launch_index
-      munged[:keypair_name]     = munged.delete(:ssh_key_name)
-      munged[:status]           = munged.delete(:state) || munged[:shutdown_state]
-      munged
-    end
-    
-    # parse IP address from aws dns name
-    def self.convert_from_ec2_dns_to_ip(str=nil)
-      return str if str.nil?
-      # if using eucalyptus we may have the raw IP already, if so just return it
-      return str if str.match( /^\d{1,3}\.\d{1,3}.\d{1,3}\.\d{1,3}$/ )
-      str.scan(/-(\d{1,3})-(\d{1,3})-(\d{1,3})-(\d{1,3})/).flatten.join('.')
-    end
-    
-    def self.parse_datetime(str)
-      Time.parse( str.chomp )
-    end
-    
-  end
-  
-end

data/lib/cloud_providers/ssh/ssh.rb

@@ -1,68 +0,0 @@
-=begin rdoc
-  This is an example CloudProvider to be used as a template for implementing new CloudProviders
-=end
-require "#{File.dirname(__FILE__)}/ssh_instance"
-
-module CloudProviders
-  class Ssh < CloudProvider
-    default_options(
-      CloudProvider.default_options.merge(
-        :hosts  => [] # list of hostnames, or addresses
-      )
-    )
-    
-    # By default we consider all hosts to be running
-    # available_hosts represents those that we are not "running" right now
-    def available_hosts
-      @available_hosts ||= []
-    end
-    
-    def running_hosts
-      hosts - available_hosts
-    end
-    
-    # Select just the instance options that are relevant to the SshInstance
-    def instance_options(opts)
-      i_opts = dsl_options.merge(opts)
-      # i_opts = SshInstance.default_options.merge(i_opts)
-      # i_opts.choose{|k,v| SshInstance.default_options.keys.include?(k)}
-    end
-    
-    # Launch a new instance
-    def run_instance(o={})
-      raise StandardError.new("You have no available hosts") if available_hosts.empty?
-      host = available_hosts.first
-      new_host = SshInstance.new(instance_options(o.merge(:name=>host)))
-      new_host.keypair
-      if new_host && new_host.refresh!
-        available_hosts.delete host
-        new_host
-      else
-        raise StandardError.new("Unable to connect to host #{host}")
-      end
-    end
-    
-    # Terminate an instance by name, or the last instance
-    def terminate_instance!(o={})
-      thost = o[:name] || running_hosts.last
-      available_hosts << thost.clone
-      SshInstance.new(instance_options(o.merge(:name=>thost, :status=>'terminated')))
-    end
-
-    # Returns an instance object for the host.  You must pass the :name option
-    def describe_instance(opts={:name=>nil})
-      raise StandardError.new("you must provide a name") unless opts[:name]
-      SshInstance.new(instance_options(o.merge(:name=>opts[:name])))
-    end
-    
-    def describe_instances(o={})
-      hosts.collect{|n| SshInstance.new(instance_options(o.merge(:name=>n)))}
-    end
-    
-    def nodes(hsh={})
-      results = running_hosts.collect{|n| SshInstance.new(instance_options(:name=>n))}
-      results.select_with_hash(hsh)
-    end
-    
-  end
-end