fairchild-poolparty 1.3.5 → 1.3.17

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 3
-:patch: 5
+:patch: 17
 :major: 1

data/bin/cloud

@@ -33,6 +33,7 @@ EOS
     $DEBUGGING      = true if command[:debug]
     $VERY_DEBUGGING = true if command[:very_debug]
     
+    PoolParty::Pool.command = command
     @loaded_pool = PoolParty::Pool.find_and_load_default_clouds_dot_rb(command[:clouds_dot_rb])
     @loaded_clouds = command[:name] ? [clouds[command[:name]]] : @loaded_pool.clouds.map {|name,cld|cld}
     @loaded_clouds.map do |cld| 

data/bin/cloud-compile

@@ -19,6 +19,7 @@ EOS
   run do |command|
     
     @loaded_clouds.each do |cld|
+      ENV["POOLPARTY_NO_VALIDATION"] = "true"
       
       cld.compile
       

data/bin/cloud-misc

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require "poolparty"
+
+require 'git-style-binary/command'
+
+GitStyleBinary.command do
+  banner <<-EOS
+Usage: #{$0} #{all_options_string}
+
+  read the code 
+EOS
+
+  short_desc "various cloud development tasks"
+  
+  opt :inst_num, "The number of the instance to run bootstrap on", :type => :int
+  opt :force, "Force rebootstrapping (useful for upgrading)", :default => false
+
+  run do |command|
+    @loaded_clouds.each do |cld|
+      cld.nodes.each do |node|
+
+        case command.argv.shift
+        when "deploy_hermes"
+          hermesdir = File.dirname(__FILE__) + "/../vendor/erlang/hermes"
+          release = `#{hermesdir}/scripts/most_recent_release tar.gz`.strip
+          node.scp(:source => release, :destination => "/var/poolparty/tmp")
+          node.scp(:source => "#{hermesdir}/scripts/target_system", :destination => "/var/poolparty/tmp")
+        end
+
+      end
+    end
+  end
+end

data/bin/cloud-show

@@ -16,6 +16,7 @@ shows output about the clouds.rb
 EOS
 
   short_desc "shows output about the clouds.rb"
+  opt :generate_graph, "Generate the dependency tree graph", :type => :string, :default => nil
 
   run do |command|
 
@@ -30,7 +31,18 @@ EOS
         "Running on: #{cld.cloud_provider_name}"
       ]
       
-      print_msg(msg)
+      if verbose?
+        msg << [
+          "Keypair: #{cld.keypair.basename}"
+        ]
+      end
+      
+      if command[:generate_graph]
+        vputs "Generating dependency graph at: #{command[:generate_graph]}"
+        cld.output_resources_graph('png', command[:generate_graph], {"fontsize" => 30})
+      end
+            
+      print_msg(msg.flatten)
       
     end
     

data/bin/cloud-ssh

@@ -13,11 +13,14 @@ EOS
 
   short_desc "ssh into an instance of the cloud"
   
+  opt :instance_number, "Ssh into the instance number", :type => :integer, :default => 0
+  
   run do |command|
     cloud = @loaded_clouds.first
     
     if !cloud.nodes.empty?
-      cloud.nodes.first.ssh
+      inst = cloud.nodes[command[:instance_number]]
+      inst ? inst.ssh : puts("Error: The instance number (#{command[:instance_number]}) is too high for the current number of instances")
     else
       puts "No running instances can be found"
     end

data/bin/cloud-thrift

@@ -22,9 +22,11 @@ Usage: #{$0} #{all_options_string}
 EOS
 
   short_desc "Start the cloud thrift interface"
-  opt :port, "Start a port", :type => :integer, :default => 11223
+  opt :port, "Start on port", :type => :integer, :default => 11223
   opt :dir, "Pid directory", :type => :string, :default => "/tmp"
   opt :daemon, "Daemonize", :type => :bool, :default => true
+  # Query params
+  opt :host, "Query on host", :type => :string, :default => "localhost"
 
   run do |command|
     
@@ -33,12 +35,12 @@ EOS
     cmd = ARGV.shift
     cmd = ARGV.shift if cmd == "thrift"
     
-    if %w(run start restart stop).include?(cmd)
+    if %w(run start restart stop query).include?(cmd)
       args = [cmd]
       args << ["--", command.argv] unless command.argv.empty?
       args.flatten!
     else
-      puts "You must pass one of the following: #{%w(run start restart stop).join(", ")}"
+      puts "You must pass one of the following: #{%w(run start restart stop query).join(", ")}"
       args = ["--help"]
     end
     
@@ -55,22 +57,34 @@ EOS
       :monitor    => true
     }.merge(command.opts)
     
-    block = Proc.new do
-      handler = CommandInterfaceHandler.new
-      processor = CloudThrift::CommandInterface::Processor.new(handler)
-      transport = Thrift::ServerSocket.new( options[:port] )
-      transportFactory = Thrift::BufferedTransportFactory.new()
-      server = Thrift::SimpleServer.new(processor, transport, transportFactory)
+    if cmd == "query"
+      require "command_query_handler"
       
-      puts "Starting the cloud server..."
-      server.serve()
-      puts "done."
-    end
-    
-    if command[:daemon]
-      Daemons.run_proc("cloud-thrift", options, &block)
-    else
-      block.call
+      @loaded_clouds.each do |cld|
+        puts CommandQueryHandler.run_query(cld, ARGV.shift, ARGV, options)
+      end
+    else      
+      block = Proc.new do
+        handler = CommandInterfaceHandler.new
+        processor = CloudThrift::CommandInterface::Processor.new(handler)
+        transport = Thrift::ServerSocket.new( options[:port] )
+        transportFactory = Thrift::BufferedTransportFactory.new()
+        server = Thrift::SimpleServer.new(processor, transport, transportFactory)
+
+        puts "Starting the cloud server..."
+        begin
+          server.serve()
+        rescue
+          exit 10
+        end
+        puts "done."
+      end
+
+      if command[:daemon]
+        Daemons.run_proc("cloud-thrift", options, &block)
+      else
+        block.call
+      end
     end
     
   end

data/config/jeweler.rb

@@ -15,7 +15,7 @@ end
     s.description = "PoolParty: The easy, open-source, cross-cloud management solution"
     s.summary = <<-EOM      
       Self-healing, auto-scaling system administration, provisioning
-      and maintaining tool that makes cloud computing easy and fun
+      and maintaining tool that makes cloud computing easier.
     EOM
     
     s.homepage = "http://poolpartyrb.com"
@@ -25,9 +25,11 @@ end
     s.test_files = Dir["test/**/test_*.rb"]
 
     s.files = (%w(Rakefile README.rdoc License.txt VERSION.yml) + Dir["{config,examples,lib,test,tasks,script,generators,bin,vendor}/**/*"])
+    s.files += ["vendor/erlang/hermes/ebin/*.tar.gz"]
     
-    s.files.exclude '**/*.beam'
-    s.files.exclude "**/*/erl_crash.dump"
+    s.files.exclude 'vendor/erlang/hermes'
+    s.files.exclude 'examples/thrift/**/*.beam'
+    # s.files.exclude "**/*/erl_crash.dump"
     
     s.has_rdoc = true
     s.extra_rdoc_files = ["README.rdoc", "License.txt", 'History.txt']

data/examples/monitored_cloud.rb

@@ -10,7 +10,7 @@ pool "poolparty" do
       image_id 'emi-39CA160F'
     end
     
-    monitor :cpu do |c|
+    monitor :'cpu-idle' do |c|
       vote_for(:expand) if c > 0.9
       vote_for(:contract) if c < 0.1
     end

data/examples/thrift/thrift_example.rb

@@ -3,7 +3,7 @@
 $:.unshift(File.dirname(__FILE__) + "/../../lib")
 $:.unshift(File.dirname(__FILE__) + "/../../examples")
 require "poolparty"
-require "simple"
+# require "simple"
 
 $:.unshift("#{File.dirname(__FILE__)}/../../lib/proto/gen-rb")
 
@@ -15,13 +15,17 @@ require "poolparty_types"
 port = ARGV.pop || 11223
 
 transport = Thrift::BufferedTransport.new(Thrift::Socket.new('localhost', port))
+# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('192.168.2.31', port))
+# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('vm', port))
 protocol = Thrift::BinaryProtocol.new(transport)
 
 client = CloudThrift::CommandInterface::Client.new(protocol)
 transport.open()
 
 cld = CloudThrift::CloudQuery.new
-cld.name = 'monitored_app'
+cld.name = 'pp2'
+# cld.name = 'monitored_app'
+# cld.name = 'vmware'
 
 resp = client.run_command(cld, "name", [])
 puts resp.response
@@ -29,8 +33,8 @@ puts resp.response
 resp = client.run_command(cld, "maximum_instances", [])
 puts resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.91"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.91"])
 p resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.01"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.01"])
 p resp.response

data/examples/vmware.rb

@@ -10,9 +10,19 @@ pool "poolparty" do
       image_id  "/Users/nmurray/Documents/VMware/Ubuntu-jaunty.vmwarevm/Ubuntu-jaunty.vmx"
       public_ip "192.168.133.128"
     end
+    has_hermes
     has_file "/etc/motd", :content => "welcome to your instance"
 
     has_convenience_helpers
   end
 
 end
+
+# hermes
+# want to be able to run cloud configure to update it
+# but want to be able to pull from locally to configure it
+# submodules?  
+# node name
+# install path
+# how to upload the escript target system
+# how to upload the tar.gz of the release

data/lib/cloud_providers/cloud_provider.rb

@@ -81,5 +81,14 @@ module CloudProviders
       results.select_with_hash(hsh)
     end
     
+    def before_compile(cld)
+    end
+    
+    def after_compile(cld)
+    end
+    
+    
+    
+    
   end
 end

data/lib/cloud_providers/cloud_provider_instance.rb

@@ -68,10 +68,15 @@ module CloudProviders
         raise StandardError.new("You must pass in a cloud to configure an instance") unless cloud
         cloud.compile(self)        
         
-        scp(:source       => keypair.full_filepath, 
-            :destination  => "/etc/poolparty/keys/#{keypair.basename}")
+        # scp(:source       => keypair.full_filepath, 
+        #     :destination  => "/etc/poolparty/keys/#{keypair.basename}")
         
         FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty" unless File.directory?(cloud.tmp_path/"etc"/"poolparty")
+        FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty"/"keys" unless File.directory?(cloud.tmp_path/"etc"/"poolparty"/"keys")
+        
+        FileUtils.cp keypair.full_filepath, cloud.tmp_path/"etc"/"poolparty"/"keys"/keypair.basename
+        File.open(cloud.tmp_path/"etc"/"poolparty"/"cloud_name", "w") {|f| f << cloud.name }
+        
         pack_clouds_dot_rb_and_expected_directories
         
         dputs("Rsyncing #{cloud.tmp_path/"*"}")
@@ -99,10 +104,10 @@ module CloudProviders
         %w(lib plugins).each do |dir|
           if File.directory?(d = cloud.clouds_dot_rb_dir/dir)
             dputs("Adding local path: #{d}")
-            FileUtils.cp_r d, cloud.tmp_path/cloud.base_config_directory
+            FileUtils.cp_r d, cloud.tmp_path/cloud.base_config_directory, :verbose => true, :remove_destination => true # req'd for symlinks
           end
         end
-        FileUtils.cp cloud.clouds_dot_rb_file, cloud.tmp_path/"/etc/poolparty/"
+        FileUtils.cp cloud.clouds_dot_rb_file, cloud.tmp_path/"/etc/poolparty/clouds.rb"
       end
       # Determine the os
       # Default to ubuntu

data/lib/cloud_providers/connections.rb

@@ -41,10 +41,11 @@ module  CloudProviders
     # {'-i'=>'keyfile, '-l' => 'fred' } would become
     # "-i keyfile -o StrictHostKeyChecking=no -i keypair.to_s -l fred"
     def ssh_options(opts={})
-      o = {"-i" => keypair.full_filepath,
+      return @ssh_options if @ssh_options && opts.empty?
+      ssh_opts = {"-i" => keypair.full_filepath,
            "-o" =>"StrictHostKeyChecking=no"
            }.merge(opts)
-      o.collect{ |k,v| "#{k} #{v}"}.join(' ')
+      @ssh_options = ssh_opts.collect{ |k,v| "#{k} #{v}"}.join(' ')
     end
     
     def rsync( opts={} )
@@ -81,15 +82,18 @@ module  CloudProviders
           while (chunk = stdin.readpartial(opts[:sysread]))
             buf << chunk
             unless chunk.nil? || chunk.empty?
-              $stdout.write(chunk) if debugging? || verbose?
+              $stdout.write(chunk) #if debugging? || verbose?
             end
           end
           err = stderr.readlines
           $stderr.write_nonblock(err)
         rescue SystemCallError => error
-          $stderr.write_nonblock(stderr)
+          err = stderr.readlines
+          $stderr.write_nonblock(err)
         rescue EOFError => error
-           # nothing
+          err = stderr.readlines
+          $stderr.write_nonblock(err)
+          # used to do nothing
         end
       end
       buf

data/lib/cloud_providers/ec2/ec2.rb

@@ -19,19 +19,22 @@ using :provider_name
 EOM
 end
 
+require "#{File.dirname(__FILE__)}/ec2_helpers"
 require "#{File.dirname(__FILE__)}/ec2_response"
 require "#{File.dirname(__FILE__)}/ec2_instance"
 
 module CloudProviders
   class Ec2 < CloudProvider
     
-    # Set the aws keys from the environment, or load from /etc/poolparty/aws.yml if the environment variable is not set
+    include CloudProviders::Ec2Helpers
+    
+    # Set the aws keys from the environment, or load from /etc/poolparty/env.yml if the environment variable is not set
     def self.default_access_key
       ENV['EC2_ACCESS_KEY'] || load_keys_from_file[:access_key]
     end
     
     def self.default_secret_access_key
-      ENV['EC2_SECRET_KEY'] || load_keys_from_file[:access_key]
+      ENV['EC2_SECRET_KEY'] || load_keys_from_file[:secret_access_key]
     end
     
     def self.default_private_key
@@ -59,10 +62,11 @@ module CloudProviders
     end
     
     # Load the yaml file containing keys.  If the file does not exist, return an empty hash
-    def self.load_keys_from_file(filename='/etc/poolparty/ec2/aws.yml', caching=true)
+    def self.load_keys_from_file(filename='/etc/poolparty/env.yml', caching=true)
       return @aws_yml if @aws_yml && caching==true
       return {} unless File.exists?(filename)
-      @aws_yml = YAML::load( open(filename).read )
+      ddputs("Reading keys from file: #{filename}")
+      @aws_yml = YAML::load( open(filename).read ) || {}
     end
     
     default_options({
@@ -87,22 +91,29 @@ module CloudProviders
         :ramdisk_id             => nil,
         :availability_zone      => nil,
         :block_device_mappings  => nil,
-        :elastic_ips            => nil, # An array of the elastic ips
-        :ebs_volume_id          => nil  # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
+        :elastic_ips            => [],  # An array of the elastic ips
+        :ebs_volumes            => []   # The volume id of an ebs volume # TODO: ensure this is consistent with :block_device_mappings
       })
       
       
     def ec2(o={})
-      @ec2 ||= Rightscale::Ec2.new(access_key, secret_access_key, o.merge(:logger => PoolParty::PoolPartyLog))
+      @ec2 ||= Rightscale::Ec2.new(access_key, secret_access_key, o.merge(:logger => PoolParty::PoolPartyLog, :endpoint_url => ec2_url))
     end
     
     # Start a new instance with the given options
     def run_instance(o={})
+      number_of_instances = o[:number_of_instances] || 1
       set_vars_from_options o
       raise StandardError.new("You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}") if !keypair_name
+      vputs("--- Launching ec2 instances")
+      vputs({
+        "image_id" => image_id,
+        "security_group" => security_group,
+        "keypair" => keypair.basename
+      })
       response_array = ec2(o).run_instances(image_id,
                                       min_count,
-                                      max_count,
+                                      number_of_instances,
                                       security_group,
                                       keypair.basename,
                                       user_data,
@@ -113,10 +124,12 @@ module CloudProviders
                                       availability_zone,
                                       block_device_mappings
                                       )
-      instances = response_array .collect do |aws_response_hash|
+      instances = response_array.collect do |aws_response_hash|
         Ec2Instance.new( Ec2Response.pp_format(aws_response_hash).merge(o) )
       end
-      #FIXME: This needs to deal with the case when an array is returned if max_instances > 1
+      
+      after_run_instance(instances)
+      
       instances.first
     end
     
@@ -153,6 +166,22 @@ module CloudProviders
        ].include?(ec2_url)
     end
     
+    # Callbacks
+    def before_compile(cld)
+    end
+    
+    def after_compile(cld)
+      save_aws_env_to_yml(cld.tmp_path/"etc"/"poolparty"/"env.yml") rescue nil
+    end
+    
+    # Run after all the instances are run
+    def after_run_instance(instances_list)
+      instances_list.each do |inst|
+        associate_address(inst.instance_id) if next_unused_elastic_ip
+        attach_volume(inst.instance_id) if next_unused_volume
+      end
+    end
+    
     # Read  yaml file and use it to set environment variables and local variables.
     def set_aws_env_from_yml_file(filename='/etc/poolparty/env.yml')
       aws = self.class.load_keys_from_file(filename)
@@ -161,8 +190,9 @@ module CloudProviders
     end
     
     # Save aws keys and env variables to a yaml file
-    def save_aws_env_to_yml(filename='/etc/poolparty/aws.yml')
-      File.open(filename, 'w') {|f| f<<YAML::dump(aws_hash) }
+    def save_aws_env_to_yml(filename='/etc/poolparty/env.yml')
+      hsh = aws_hash(default_options, "/etc/poolparty/ec2")
+      File.open(filename, 'w') {|f| f<<YAML::dump(hsh) }
     end
     
     # Return a hash of the aws keys and environment variables