auser-poolparty 1.1.5 → 1.1.6

data/README.txt

@@ -37,18 +37,6 @@ Erlang
 
 sudo gem install auser-poolparty
 
-== TODO:
-* Update hot reloading of puppet (force-reload?)
-* Add pingback for nodes to master when failure occurs
-* Add become master for the nodes
-* Add hot reload to haproxy while reconfiguring
-* Replace services with Runit
-* Refactor provisioning to use erlang
-* Add queuing of tasks on the messenger
-* Add syntax checker on the manifests
-* Move in collectd support
-* Remove master-slave protocol
-
 == LICENSE:
 
 (The MIT License)

data/Rakefile

@@ -1,4 +1,11 @@
 require 'config/requirements'
+
+begin
+  require 'hanna/rdoctask'
+rescue Exception => e
+  require "rake/rdoctask"
+end
+
 require 'config/jeweler' # setup gem configuration
 
 Dir['tasks/**/*.rake'].each { |rake| load rake }
@@ -42,4 +49,13 @@ namespace :gem do
   task :test_then_install => [:spec, :install]
 end
 
-task :release => [:update_timestamp]
+task :release => [:update_timestamp]
+
+
+# Generate documentation
+Rake::RDocTask.new do |rd|
+  rd.main = "Readme.txt"
+  rd.rdoc_files.include("Readme.txt", "lib/**/*.rb")
+  rd.rdoc_dir = "rdoc"
+  # rd.template = "hanaa"
+end

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 5
-:major: 1
 :minor: 1
+:patch: 6
+:major: 1

data/bin/cloud

@@ -15,7 +15,26 @@ new_args = new_args.reject {|arg| Binary.available_binaries_for("cloud").include
 # Let's make sure if no command is passed in that we show the help message
 new_args.push("-h") unless name
 
-o = PoolParty::Optioner.new(new_args, {:extra_help => "\nCloud actions\n#{Binary.list_binaries_for("cloud")}\n\n", :abstract => true}) do |opts, optioner|  
+# Binary.list_binaries_for("cloud")
+extra_help_string = <<-EOE
+
+  Cloud commands
+    cloud start       starts a single instance in your cloud.
+    cloud bootstrap   bootstraps an instance or multiple instances
+    cloud configure   configures an instance or multiple instances
+    cloud provision   combines bootstrap and configures an instance or multiple instances
+    cloud expand      expands the cloud by a single instance
+    cloud contract    contracts the cloud by a single instance
+    cloud describe    show the configuration of the current clouds.rb configuration file
+    cloud list        show a list of the current instances on the cloud
+    cloud ssh         ssh into an instance of the cloud
+    cloud terminate   terminate the cloud
+    
+EOE
+
+o = PoolParty::Optioner.new(new_args, {
+  :extra_help => extra_help_string,
+  :abstract => true}) do |opts, optioner|  
   opts.on('-n cloudname', '--name name', 'Address this cloud')    { |c| optioner.cloudname c }
 end
 

data/bin/cloud-bootstrap

@@ -3,19 +3,29 @@ $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 require "poolparty"
 require "poolparty/provision/boot_strapper"
 
+
 o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
+  opts.on('-i num', '--num i', 'Instance number')    { |i| optioner.inst_num i.to_i }
 end
 
 o.loaded_clouds.each do |cld|
   # If an IP or DNS name is given, bootstrap that node, otherwise, bootstrap all running nodes.
-  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.running_instance_ips
+  # ::FileUtils.rm_rf "#{Default.tmp_path}/" unless o.testing?
+  
+  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.nodes(:status => "running").collect {|i| i.ip }.sort.compact
   if nodes.empty?
     puts "No nodes to bootstrap"
   else
-    nodes.each do |address|
-      puts "\nBootStrapping: #{address}\n--------------------"
-      ::PoolParty::Provision::BootStrapper.new address, {:full_keypair_path => cld.keypair.full_filepath, :cloud => cld}
+    if o.inst_num?
+      vputs "\nBootStrapping: #{nodes[o.inst_num]}\n--------------------"
+      ::PoolParty::Provision::BootStrapper.new nodes[o.inst_num], :cloud => cld
+    else
+      nodes.each do |address|
+        vputs "\nBootStrapping: #{address}\n--------------------"      
+        ::PoolParty::Provision::BootStrapper.new address, :cloud => cld
+      end
     end
   end
 end
+

data/bin/cloud-configure

@@ -2,7 +2,17 @@
 $:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
 require "poolparty"
 
-o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+extra_help_string = <<-EOE
+
+  Cloud configure
+  
+    Cloud configure will repackage the deployment and send it off to a 
+    remote instance from within the cloud. It then will reconfigure
+    itself and alert the rest of the cloud
+    
+EOE
+
+o = PoolParty::Optioner.new(ARGV, {:extra_help => extra_help_string}) do |opts, optioner|
   opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| optioner.cloudname c }
   opts.on('-i num', '--num i', 'Instance number')    { |i| optioner.inst_num i.to_i }
 end
@@ -11,16 +21,21 @@ o.loaded_clouds.each do |cld|
   # If an IP or DNS name is given, bootstrap that node, otherwise, bootstrap all running nodes.
   # ::FileUtils.rm_rf "#{Default.tmp_path}/" unless o.testing?
   
-  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.remote_instances_list.collect {|inst| inst.ip if inst.running? }.compact
-  if nodes.empty?
-    puts "No nodes to configure"
+  if cld.nodes(:status => "running").size < 1
+    puts header("No instances running")
   else
-    if o.inst_num?
-      ::PoolParty::Provision::DrConfigure.new( nodes[o.inst_num], :full_keypair_path => cld.keypair.full_filepath,  :cloud => cld )
+    nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.nodes(:status => "running").collect {|i| i.ip }.sort.compact
+    if nodes.empty?
+      puts "No nodes to configure"
     else
-      nodes.each do |address|
-        puts "Configuring: #{address}"
-        ::PoolParty::Provision::DrConfigure.new( address, :full_keypair_path => cld.keypair.full_filepath,  :cloud => cld )
+      if o.inst_num?
+        vputs "Configuring #{nodes[o.inst_num]}"
+        ::PoolParty::Provision::DrConfigure.new( nodes[o.inst_num], :cloud => cld )
+      else
+        nodes.each do |address|
+          puts "Configuring: #{address}"
+          ::PoolParty::Provision::DrConfigure.new( address, :cloud => cld )
+        end
       end
     end
   end

data/bin/cloud-contract

@@ -7,11 +7,11 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n name', '--name name', 'Listing for cloud name')    { |c| optioner.cloudname c }
 end
 
-o.loaded_clouds.each do |cl|
-  with_cloud(cl) do
-    puts header("Contracting cloud #{name}")
-    puts "#{list_of_running_instances.size} running instances (#{minimum_instances} minimum instances)"
-    puts testing ? "Not contracting (test mode)" : "Contracting the cloud if possible"
-    contract_cloud_if_necessary( !testing )
-  end
+o.loaded_clouds.each do |cld|  
+  puts header("Contracting cloud #{cld.name}")
+  puts "#{cld.nodes(:status => "running").size} running instances (#{cld.minimum_instances} minimum instances)"
+  puts testing ? "Not contracting (test mode)" : "Contracting the cloud if possible"
+
+  last_instance_id = cld.nodes(:status => "running").last[:instance_id]
+  cld.remote_base.terminate_instance! cld.options.merge(:instance_id => last_instance_id)
 end

data/bin/cloud-describe

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+
+o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
+  opts.on('-n name', '--name name', 'Listing for cloud name')    { |c| optioner.cloudname c }
+end
+
+o.loaded_clouds.each do |cld|
+  puts "\nFrom #{$pool_specfile}\n"
+  print_with_nice_printer "Cloud: #{cld.name}", [
+    "instances #{cld.minimum_instances}..#{cld.maximum_instances}",
+    "keypair #{cld.keypair}",
+    "remoter #{cld.remote_base.class.to_s.top_level_class}"
+  ] do |printer|
+    if cld.nodes(:status => "running").size > 0
+      printer.empty
+      printer.center("Active instances")
+      printer << cld.nodes(:status => "running").map{|a| a.ip.strip }.join("\n")
+    end
+    if cld.nodes(:status => "pending").size > 0
+      printer.empty
+      printer.center "Pending instances"
+      printer << cld.nodes(:status => "pending").map{|a| a.to_s}.join("\n")
+    end
+    printer.empty
+  end
+end

data/bin/cloud-expand

@@ -7,13 +7,20 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n name', '--name name', 'Expand the cloud on this name')    { |c| optioner.cloudname c }
 end
 
-o.loaded_clouds.each do |cloud|
+o.loaded_clouds.each do |cld|
 
-  with_cloud(cloud) do
-    vputs header("Expanding cloud #{name}")
-    vputs "#{list_of_running_instances.size} running instances of #{maximum_instances} possible instances"
-    vputs testing ? "Not expanding (test mode)" : "Expanding the cloud"
-    expand_cloud_if_necessary( !testing )
+  vputs header("Expanding cloud #{cld.name}")
+  vputs "#{cld.nodes(:status => "running").size} running instances of #{cld.maximum_instances} possible instances"
+  vputs testing ? "Not expanding (test mode)" : "Expanding the cloud"
+  
+  cld.launch_instance! do |node|
+    ::PoolParty::Provision::BootStrapper.new(node.ip, :cloud => cld)
+    ::PoolParty::Provision::DrConfigure.new(node.ip,  :cloud => cld)
+    
+    puts <<-EOM
+      Your cloud has expanded. The new ip is #{node.ip}
+    EOM
   end
   
+  
 end

data/bin/cloud-list

@@ -10,14 +10,12 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
 end
 
 o.loaded_clouds.each do |cld|
-  
   puts header("Listing cloud #{cld.name}")
-  
   puts subheader("Active instances")
-  puts cld.list_of_running_instances.map{|a| a.to_s}.join("\n")
+  puts cld.nodes(:status => "running").map{|a| "#{a[:instance_id]}\t#{a[:ip]}" }.join("\n")
     
-  if cld.list_of_pending_instances.size > 0
+  if cld.nodes(:status => "pending").size > 0
     puts subheader("Pending instances")
-    puts cld.list_of_pending_instances.map{|a| a.to_s}.join("\n")
+    puts cld.nodes(:status => "pending").map{|a| "#{a[:instance_id]}\t#{a[:ip]}" }.join("\n")
   end
 end

data/bin/cloud-osxcopy

@@ -9,7 +9,7 @@ end
 
 o.loaded_clouds.each do |cld|
   
-  node = cld.list_of_running_instances.first
+  node = cld.nodes(:status => "running").first
   ip = node.ip
   puts "ip: #{ip}"
   Kernel.system "echo #{ip} | pbcopy"

data/bin/cloud-provision

@@ -10,7 +10,7 @@ end
 
 o.loaded_clouds.each do |cld|
   # If an IP or DNS name is given, bootstrap that node, otherwise, bootstrap all running nodes.
-  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.remote_instances_list.collect {|inst| inst.ip if inst.running? }.compact
+  nodes = !o.unflagged_args.empty? ?  o.unflagged_args : cld.nodes(:status => "running").collect {|i|i.ip }.compact
   if nodes.empty?
     puts "No nodes to bootstrap"
   else

data/bin/cloud-show

@@ -10,17 +10,18 @@ o.loaded_clouds.each do |cld|
   puts "\nFrom #{$pool_specfile}\n"
   print_with_nice_printer "Cloud: #{cld.name}", [
     "instances #{cld.minimum_instances}..#{cld.maximum_instances}",
-    "keypair #{cld.keypair}"
+    "keypair #{cld.keypair}",
+    "using #{cld.remote_base.class.to_s.top_level_class}"
   ] do |printer|
-    if cld.list_of_running_instances.size > 0
+    if cld.nodes(:status => "running").size > 0
       printer.empty
       printer.center("Active instances")
-      printer << cld.list_of_running_instances.map{|a| a.ip.strip }.join("\n")
+      printer << nodes(:status => "running").map{|a| a.ip.strip }.join("\n")
     end
-    if cld.list_of_pending_instances.size > 0
+    if cld.nodes(:status => "pending").size > 0
       printer.empty
       printer.center "Pending instances"
-      printer << cld.list_of_pending_instances.map{|a| a.to_s}.join("\n")
+      printer << cld.nodes(:status => "pending").map{|a| a.to_s}.join("\n")
     end
     printer.empty
   end

data/bin/cloud-ssh

@@ -11,9 +11,7 @@ end
 load_pool(o.spec? ? o.spec : Binary.get_existing_spec_location)
 
 @cloud = o.loaded_clouds.first
-nodes = @cloud.list_of_running_instances
-
-puts nodes.map {|a| a.to_s}.join("\n")
+nodes = @cloud.nodes(:status => "running")
 
 if !nodes.empty?
   n = o.inst_num? ? o.inst_num : 0

data/bin/cloud-start

@@ -7,11 +7,13 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
 end
 
 o.loaded_clouds.each do |cld|
-  if cld.list_of_running_instances.size.zero?
-    puts header("Starting cloud #{cld.name}")
-    puts "#{cld.list_of_running_instances.size} running instances (#{cld.minimum_instances} - #{cld.maximum_instances})"
+
+  if cld.nodes(:status => "running").size.zero?
+    puts header("Starting cloud #{cld.name} (#{cld.keypair})")
+    puts "#{cld.nodes(:status => "running").size} running instances (#{cld.minimum_instances} - #{cld.maximum_instances})"
     
-    cld.launch_instance! do |node|
+    cld.launch_instance!(cld.dsl_options) do |node|
+      
       ::PoolParty::Provision::BootStrapper.new(node.ip, :cloud => cld)
       ::PoolParty::Provision::DrConfigure.new(node.ip,  :cloud => cld)
       

data/bin/cloud-terminate

@@ -7,14 +7,16 @@ o = PoolParty::Optioner.new(ARGV) do |opts, optioner|
   opts.on('-n name', '--name name', 'Terminate the cloud of this name')    { |c| optioner.cloudname c }
 end
 
-o.loaded_clouds.each do |cloud|
-  
-  with_cloud(cloud) do
+o.loaded_clouds.each do |cld|
+  cld.nodes(:status => "running").each do |inst|
+    msg = <<-EOM
+Shutting down #{inst.instance_id}
+Are you sure? (Y/N)
+    EOM
     
-    list_of_running_instances.each do |inst|
+    if are_you_sure?(msg)
       puts "Shutting down #{inst.instance_id}"
-      terminate_instance!(:instance_id => inst.instance_id)
+      cld.terminate_instance!(:instance_id => inst.instance_id)
     end
-    
-  end    
+  end
 end

data/bin/server-cloud-elections

@@ -19,11 +19,14 @@ o.loaded_clouds.each do |cld|
   
   case vote_for
   when "expand"
-    # PoolParty.log << "Launching new instance"
+    vputs "Launching new instance"
     cld.send :launch_instance!, schema.options do |node|
+      dputs "Started node at #{node[:ip]}"
+      
       ::PoolParty::Provision::BootStrapper.new( node[:ip], :cloud => cld )
       ::PoolParty::Provision::DrConfigure.new( node[:ip], :cloud => cld )
     end
+    
   when "contract"
     last_instance_id = %x[/usr/bin/server-list-active instance_id].split("\t").last
     remoter_base.terminate_instance! schema.options.merge(:instance_id => last_instance_id)

data/bin/server-list-active

@@ -8,18 +8,22 @@ json_file = ARGV[0] || PoolParty::Default.properties_hash_file
 raise "ERROR: no json_file given" if !json_file
 schema = PoolParty::Schema.new( ::File.read(json_file) )
 
-remoter_base = schema.options.remote_base
+inst_array = PoolParty::Neighborhoods.load_default.instances.instances
 
-# TODO: WTF? Change this
-instances = PoolParty::Remote::Ec2.describe_instances(schema.options)
-inst_array = instances.map {|inst| inst.send list_type.to_sym }
+if !inst_array || inst_array.empty?
+  remoter_base = schema.options.remote_base
+  remote_base = remoter_base.top_level_class.constantize(PoolParty::Remote)
+
+  instances = remote_base.send :describe_instances, schema.options
+  inst_array = instances.map {|inst| inst.send list_type.to_sym }  
+end
 
 print_array = 
 case list_type
 when "ip"
-  inst_array.select {|a| a =~ /\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/}
+  inst_array.compact.map {|i| i.ip }
 when "internal_ip"
-  inst_array.compact.map {|ip| ip.gsub(/ip-/, '').gsub(/\.ec2\.internal/, '').gsub(/-/, '.') }
+  inst_array.compact.map {|i| i.internal_ip || i.ip }
 else
   inst_array
 end

data/bin/server-monitor.ru

@@ -1,18 +1,35 @@
 #!/usr/bin/env ruby
 require 'rubygems'
 require 'rack'
-require 'rack/contrib'
+# require 'rack/contrib'
 require 'json'
 require 'thin'
 require ::File.join(::File.dirname(__FILE__),'..','lib/poolparty/monitors/', 'monitor_rack.rb') 
 
+class Baton
+  def initialize(app)
+    @app = app
+  end
+  def call(env)
+    self.class.call(env)
+  end
+  def self.call(env)
+    puts "received: #{env.to_yaml}\n------\n"
+    response = Rack::Response.new
+    response.write "requested: #{env["REQUEST_PATH"]}"
+    response.finish
+  end
+end
+
 
 app = Rack::Builder.new do
   use Rack::Reloader, 2
   use Rack::ShowExceptions
   use Rack::Lint
+  use Baton
   # use Rack::PostBodyContentTypeParser  #parses json requests to params hash
   run Monitors::MonitorRack.new()
+  run_after
 end
 
 if __FILE__ == $0