fairchild-poolparty 1.1.4 → 1.1.5

data/lib/poolparty/provision/boot_strapper.rb

@@ -1,42 +1,39 @@
+=begin rdoc
+  BootStrapper is contains the basic files that need to be uploaded to a new instance and the first commands that are run to set up a base PoolParty enviornment.  This includes things like ensuring that ruby, rubygems, and poolparty are installed, and then start the monitor.
+=end
+
 require "#{::File.dirname(__FILE__)}/../net/remoter/connections"
 
-#provide a very simple provisioner with as few dependencies as possible
+# Provide a very simple provisioner with as few dependencies as possible
 module PoolParty
   module Provision
- 
+  
     class BootStrapper
       include ::PoolParty::Remote
       
       # List of gems that are default to install
       def self.gem_list
-        @gem_list ||= %w( flexmock
-                          lockfile
-                          logging
-                          ZenTest
+        @gem_list ||= %w( logging
                           rake
                           xml-simple
-                          sexp_processor
                           net-ssh
                           net-sftp
                           net-scp
                           net-ssh-gateway
-                          echoe
                           highline
                           json
-                          capistrano
-                          ParseTree
-                          ruby2ruby
                           activesupport
                           grempe-amazon-ec2
-                          RubyInline
+                          ohai
                           chef
                           auser-dslify
-                          auser-butterfly
                           auser-parenting
-                          thin                          
+                          adamwiggins-rest-client
+                          rack
+                          thin
                         )
       end
-  
+
       # Default options for the boot_strapper
       @defaults = ::PoolParty::Default.default_options.merge({
         :full_keypair_path   => "#{ENV["AWS_KEYPAIR_NAME"]}" || "~/.ssh/id_rsa",
@@ -45,34 +42,50 @@ module PoolParty
       })
       class <<self; attr_reader :defaults; end
       
-      def initialize(host, opts={}, &block)
+      # In case the method is being called on ourself, let's check the 
+      # defaults hash to see if it's available there
+      def method_missing(m,*a,&block)
+        if self.class.defaults.has_key?(m) 
+          self.class.defaults[m]
+        elsif @cloud
+          @cloud.send m, *a, &block
+        else
+          super
+        end
+      end
+      
+      def initialize(host, opts={}, &block)        
         self.class.defaults.merge(opts).to_instance_variables(self)
         @target_host = host
         @cloud = opts[:cloud]
                 
-        instance_eval &block if block        
+        instance_eval &block if block
         @cloud.call_before_bootstrap_callbacks if @cloud
         
         default_commands
+        dputs "Starting bootstrapping process on #{host}"
         execute!
-
+        
         @cloud.call_after_bootstrap_callbacks if @cloud
         after_bootstrap
+        dputs "Bootstrapping complete on #{host}"
       end
       
       def self.class_commands
         @class_commands ||= []
       end
         
+      # Collect all the bootstrap files that will be uploaded to the remote instances
       def pack_the_dependencies
         # Add the keypair to the instance... shudder
-        ::Suitcase::Zipper.add(@cloud.keypair, "keys")
+        ::Suitcase::Zipper.add(keypair, "keys")
         
         # Use the locally built poolparty gem if it is availabl
         if edge_pp_gem = Dir["#{Default.vendor_path}/../pkg/*poolparty*gem"].pop
           puts "using edge poolparty: #{::File.expand_path(edge_pp_gem)}"
-          ::Suitcase::Zipper.add(edge_pp_gem, 'gems')        
+          ::Suitcase::Zipper.add(edge_pp_gem, 'gems')
         else
+          vputs "using gem auser-poolparty. use rake build to use edge"
           self.class.gem_list << 'auser-poolparty'
         end
         # Add the gems to the suitcase
@@ -81,7 +94,7 @@ module PoolParty
 
         ::Suitcase::Zipper.packages( "http://rubyforge.org/frs/download.php/45905/rubygems-1.3.1.tgz",
                  "#{Default.tmp_path}/trash/dependencies/packages")
-        ::Suitcase::Zipper.add("templates/")
+        # ::Suitcase::Zipper.add("templates/")
         
         ::Suitcase::Zipper.add("#{::File.dirname(__FILE__)}/../templates/monitor.ru", "/etc/poolparty/")
         ::Suitcase::Zipper.add("#{::File.dirname(__FILE__)}/../templates/monitor.god", "/etc/poolparty/")
@@ -89,11 +102,20 @@ module PoolParty
         ::Suitcase::Zipper.add("#{Default.tmp_path}/trash/dependencies/cache", "gems")        
         
         ::Suitcase::Zipper.add("#{::File.join(File.dirname(__FILE__), '..', 'templates', 'gemrc' )}", "etc/poolparty")
+        
+        instances = @cloud.nodes(:status => "running") + [@cloud.started_instance]
+        ::Suitcase::Zipper.add_content_as(
+          {:instances => instances.flatten.compact}.to_json, 
+          "neighborhood.json", "/etc/poolparty")
+        
         ::Suitcase::Zipper.build_dir!("#{Default.tmp_path}/dependencies")
         
+        ::Suitcase::Zipper.flush!
+        
         # ::FileUtils.rm_rf "#{Default.tmp_path}/trash"
       end
   
+      # The commands to setup a PoolParty enviornment
       def default_commands
         pack_the_dependencies
         ::FileUtils.rm_rf "#{Default.tmp_path}/dependencies/gems/cache"
@@ -107,7 +129,7 @@ module PoolParty
           'cd /var/poolparty/dependencies',
           "cp /var/poolparty/dependencies/etc/poolparty/gemrc /etc/poolparty",          
           "#{installer} update",
-          "#{installer} install -y ruby1.8 ruby1.8-dev libopenssl-ruby1.8 build-essential wget",  #optional, but nice to have
+          "#{installer} install -y ruby ruby1.8-dev libopenssl-ruby1.8 build-essential wget",  #optional, but nice to have
           "tar -zxvf packages/rubygems-1.3.1.tgz",
           "cd rubygems-1.3.1",
           "ruby setup.rb --no-ri --no-rdoc",
@@ -121,9 +143,10 @@ module PoolParty
           'touch /var/poolparty/POOLPARTY.PROGRESS',
           "mkdir -p /root/.ssh",
           "cp /var/poolparty/dependencies/keys/* /root/.ssh/",
-          "chmod 600 /root/.ssh/#{@cloud.keypair.basename}",
+          "chmod 600 /root/.ssh/#{::File.basename(@cloud.keypair.full_filepath)}",
           # "god -c /etc/poolparty/monitor.god",
-          "thin -R /etc/poolparty/monitor.ru -p 8642 --daemon --pid /var/run/stats_monitor.pid start 2>/dev/null",
+          "mkdir -p /var/log/poolparty/",
+          "thin -R /etc/poolparty/monitor.ru -p 8642 --pid /var/run/stats_monitor.pid --daemon -l /var/log/poolparty/monitor.log start 2>/dev/null",
           'echo "bootstrap" >> /var/poolparty/POOLPARTY.PROGRESS']
         commands << self.class.class_commands unless self.class.class_commands.empty?
       end

data/lib/poolparty/provision/configurations/chef.rb

@@ -5,15 +5,18 @@ module PoolParty
       def self.commands
         [
           "mkdir -p /etc/chef/cookbooks /etc/chef/cache",
-          "cp -R /var/poolparty/dr_configure/chef/recipes/* /etc/chef/cookbooks",
-          "cp /var/poolparty/dr_configure/chef_config.rb /etc/chef/solo.rb",
+          "cp -R /var/poolparty/dr_configure/chef/cookbooks/* /etc/chef/cookbooks",
+          "cp /var/poolparty/dr_configure/chef/solo.rb /etc/chef/solo.rb",
+          "cp /var/poolparty/dr_configure/chef/dna.json /etc/chef/dna.json",
           "/usr/bin/chef-solo -c /etc/chef/solo.rb -j /etc/chef/dna.json"
         ]
       end
       def self.files_to_upload
         [ 
-          "#{Default.tmp_path}/chef_config.rb",
-          "#{Default.tmp_path}/dna.json"
+          "#{Default.tmp_path}/dr_configure/chef/solo.rb",
+          "#{Default.tmp_path}/dr_configure/chef/dna.json",
+          "#{Default.base_config_directory}/chef/solo.rb",
+          "#{Default.base_config_directory}/chef/dna.json"
         ]
       end
     end

data/lib/poolparty/provision/dr_configure.rb

@@ -10,64 +10,119 @@ module PoolParty
     class DrConfigure
      include ::PoolParty::Remote
      
-     def self.defaults 
+     def self.defaults
        ::PoolParty::Default.default_options.merge({
          :full_keypair_path   => "#{ENV["AWS_KEYPAIR_NAME"]}" || "~/.ssh/id_rsa",
          :installer           => 'apt-get install -y',
-         :dependency_resolver => 'chef'
+         :dependency_resolver => 'chef',
+         :invalid_run_count => 2
        })
      end
-          
-     attr_accessor :full_keypair_path
+     
+     # In case the method is being called on ourself, let's check the 
+     # defaults hash to see if it's available there
+     def method_missing(m,*a,&block)
+       if self.class.defaults.has_key?(m) 
+         self.class.defaults[m]
+       elsif @cloud
+         @cloud.send m, *a, &block
+       else
+         super
+       end
+     end
+    
+     attr_reader :cloud, :keypair, :run_count
+     
      def initialize(host, opts={}, &block)
        self.class.defaults.merge(opts).to_instance_variables(self)
        @target_host = host
        @configurator = "::PoolParty::Provision::#{dependency_resolver.capitalize}".constantize
        @cloud = opts[:cloud]
+       @keypair = @cloud.keypair
+       @run_count = 0
        
        @cloud.call_before_configure_callbacks if @cloud
        prescribe_configuration
        execute!
      end
      
+     private
+     
      def prescribe_configuration
+       ::FileUtils.mkdir_p "#{Default.tmp_path}/dr_configure" unless ::File.directory?("#{Default.tmp_path}/dr_configure")
       ::File.cp $pool_specfile, "#{Default.tmp_path}/dr_configure/clouds.rb"
       ::File.open "#{Default.tmp_path}/dr_configure/clouds.json", "w" do |f|
         f << cloud.to_properties_hash.to_json
       end
-            
+      
       setup_configurator
-      write_erlang_cookie
+      # write_erlang_cookie
       @configurator.files_to_upload.each {|f| ::FileUtils.cp f, "#{Default.tmp_path}/dr_configure/#{::File.basename(f)}" if ::File.file?(f) }
       
       pack_up_and_ship_off_suitcase
-                  
-      commands << [
-        'chmod 600 /var/poolparty/dr_configure/clouds.json',
-        'chmod 600 /var/poolparty/dr_configure/clouds.rb',
-        'cp -f /var/poolparty/dr_configure/clouds.json /etc/poolparty',
-        'cp /var/poolparty/dr_configure/clouds.rb /etc/poolparty',
-        'cp /var/poolparty/dr_configure/erlang.cookie /root/.erlang.cookie',        
-        'ruby /var/poolparty/dr_configure/erlang_cookie_maker',
-        "touch /var/poolparty/POOLPARTY.PROGRESS",
-        'echo "configure" >> /var/poolparty/POOLPARTY.PROGRESS'
-        ]
-      commands << self.class.class_commands unless self.class.class_commands.empty?
-      commands << @configurator.commands      
+      run_commands          
+     end
+     
+     def execute!
+      super
+      add_run_count!
+      begin
+        if @cloud.verifiers.size > 0
+          @cloud.passing?
+          @cloud.vputs "Cloud passed verification"
+        end
+      rescue Exception => e
+        if run_count < invalid_run_count
+          @cloud.vputs <<-EOM
+
+          Verification failed: #{e}"
+            Running configure again to try to solve the problem
+
+          EOM
+          execute!
+        else
+          @cloud.vputs <<-EOM
+
+          Verification failed: #{e}"
+            Please check your clouds.rb for any errors
+
+          EOM
+        end        
+      end
+     end
+     
+     def run_commands
+       commands << [
+         'chmod 644 /var/poolparty/dr_configure/clouds.json',
+         'chmod 644 /var/poolparty/dr_configure/clouds.rb',
+         'cp /var/poolparty/dr_configure/clouds.json /etc/poolparty',
+         'cp /var/poolparty/dr_configure/clouds.rb /etc/poolparty',
+         "touch /var/poolparty/POOLPARTY.PROGRESS",
+         'echo "configure" >> /var/poolparty/POOLPARTY.PROGRESS'
+         ]
+       commands << self.class.class_commands unless self.class.class_commands.empty?
+       commands << @configurator.commands
      end
      
      def pack_up_and_ship_off_suitcase
        ::Suitcase::Zipper.build_dir!("#{Default.tmp_path}/dr_configure")
-       
-       rsync "#{Default.tmp_path}/dr_configure/", "/var/poolparty/dr_configure/"
+       rsync "#{Default.tmp_path}/dr_configure/", "/var/poolparty/dr_configure/", ['-a --stats']
      end
      
+     # Pack up  monitors verifiers plugins directories in the same direcotry as your clouds.rb and send to nodes.
      def setup_configurator
        # @cloud.write_properties_hash("#{Default.tmp_path}/properties_hash.rb")
-       #TODO: move to puppet class
-       puts "writting new config file"
-       @cloud.build_and_store_new_config_file("#{Default.tmp_path}/dr_configure/poolparty.pp") 
-       # Neighborhoods.clump(@cloud.remote_instances_list, "#{Default.tmp_path}/neighborhood.json")
+       #TODO: move to puppet class       
+       #TODO: remove or conditionalize this puppet specific task
+       @cloud.build_and_store_new_config_file("#{Default.tmp_path}/dr_configure/poolparty.pp")
+       
+       %w(monitors verifiers plugins).each do |dir|
+        @cloud.pack_user_directory dir
+       end
+     end
+     
+     def add_run_count!
+      @run_count += 1
      end
      
      def write_erlang_cookie

data/lib/poolparty/resources/cron.rb

@@ -7,6 +7,14 @@ module PoolParty
         :command => nil,
         :user => "root"
       })
+      
+      def present
+        :create
+      end
+      
+      def absent
+        :delete
+      end
 
     end
     

data/lib/poolparty/resources/directory.rb

@@ -4,7 +4,7 @@ module PoolParty
     class Directory < Resource
       
       def present
-        "directory"
+        :create
       end
       
     end

data/lib/poolparty/resources/file.rb

@@ -3,6 +3,18 @@ module PoolParty
         
     class File < Resource
       
+      def loaded(o={}, &block)
+        parent.has_directory ::File.dirname(name) if parent
+      end
+      
+      def present
+        :create
+      end
+      
+      def absent
+        :delete
+      end
+      
       def after_create
         run_render = dsl_options.include?(:render_as) ? dsl_options.delete(:render_as) : false
         
@@ -23,6 +35,10 @@ module PoolParty
         end
       end
       
+      def method_missing m, *a, &block
+        super rescue ::File.send(m, *a, &block)
+      end
+      
     end
     
   end

data/lib/poolparty/resources/package.rb

@@ -3,8 +3,13 @@ module PoolParty
         
     class Package < Resource
       
-      default_options({
-      })
+      def present
+        :install
+      end
+      
+      def absent
+        :remove
+      end
       
     end
     

data/lib/poolparty/resources/service.rb

@@ -4,16 +4,17 @@ module PoolParty
     class Service < Resource
       
       default_options({
-        :ensures => "running",
+        :ensures => "enable",
         :enable => true
       })
       
       def present
-        "running"
+        :start
       end
       def absent
-        "stopping"
+        :stop
       end
+      
     end
     
   end

data/lib/poolparty/resources/symlink.rb

@@ -6,14 +6,6 @@ module PoolParty
       def initialize *args, &block
         super
       end
-            
-      def source(i=nil)
-        i ? options[:ensures] = i : options[:ensures]
-      end
-      
-      def present
-        options[:ensures]
-      end
       
     end
     

data/lib/poolparty/schema.rb

@@ -1,3 +1,8 @@
+=begin rdoc
+  Schema
+  
+  Schemas are basically stub clouds without the PoolParty stack
+=end
 module PoolParty
   class Schema
     attr_accessor :hsh
@@ -6,6 +11,8 @@ module PoolParty
       case h
       when Hash
         h.each {|k,v| self[k] = v}
+      when Array
+        h.each {|el| self[el['ip']]=el}
       when String        
         JSON.parse(h).each {|k,v| self[k.to_sym] = v}
       end
@@ -40,25 +47,33 @@ module PoolParty
         @hsh.send(sym, *args, &block)
       end
     end
+    
+    def to_cloud(node={})
+      require "ostruct"
+      
+      $pool_specfile = "/etc/poolparty/clouds.rb"
+      
+      remoter_base = PoolParty::Remote.module_eval(options.remote_base.split('::')[-1].camelcase)
+      # TODO: Seriously, make this sexier
+      
+      cld = OpenStruct.new(options)
+      cld.keypair = ::PoolParty::Key.new("/etc/poolparty/#{node[:keypair]}")
+      cld.remoter_base = remoter_base
+      cld.build_and_store_new_config_file = "/etc/poolparty/clouds.json"
+      cld.dependency_resolver = PoolParty.module_eval(options.dependency_resolver.split("::")[-1].camelcase).send(:new)
+      
+      cld
+    end
   end
 end
-# class Hash
-#   def [](k=nil)
-#     if self.has_key?(k.to_sym)
-#       fetch(k.to_sym)
-#     elsif self.has_key?(k.to_s)
-#       fetch(k.to_s)
-#     else
-#       nil
-#     end
-#   end
-#   def method_missing(sym, *args, &block)
-#     if has_key?(sym.to_sym)
-#       fetch(sym)
-#     elsif has_key?(sym.to_s)
-#       fetch(sym.to_s)
-#     else
-#       super
-#     end
-#   end  
-# end
+class Hash
+  def method_missing(sym, *args, &block)
+    if has_key?(sym.to_sym)
+      fetch(sym)
+    elsif has_key?(sym.to_s)
+      fetch(sym.to_s)
+    else
+      super
+    end
+  end  
+end

data/lib/poolparty/templates/haproxy.conf

@@ -5,32 +5,34 @@ global
   user          haproxy
   group         haproxy
   chroot        /var/run/haproxy
-  log 127.0.0.1 local0 debug
-  pidfile /var/run/haproxy.pid
+  log           127.0.0.1 local0 debug
+  pidfile       /var/run/haproxy.pid
  
 defaults  
-  log               global # Log to the global log
-  clitimeout 150000
-  srvtimeout 150000
-  contimeout 30000
- 
-  option            httplog
-  option            httpclose     # disable keepalive (HAProxy does not yet support the HTTP keep-alive mode)
-  option            forwardfor    # enable insert of X-Forwarded-For headers
-  retries 3
-  option            redispatch
-  
-  balance roundrobin            # each server is used in turns, according to assigned weight
-  
-  # Stats
-  stats enable
-  stats uri /poolparty
-  stats realm Statistics\ for\ PoolParty
+  log           global # Log to the global log
+                
+  clitimeout    150000
+  srvtimeout    150000
+  contimeout    30000
+                
+  option        httplog
+  option        httpclose     # disable keepalive (HAProxy does not yet support the HTTP keep-alive mode)
+  option        forwardfor    # enable insert of X-Forwarded-For headers
+  option        redispatch
+
+  retries       3
+                
+  balance       roundrobin    # each server is used in turns, according to assigned weight
+                
+  # Stats       
+  stats         enable
+  stats uri     /poolparty
+  stats realm   Statistics\ for\ PoolParty
 
 listen <%= @node[:poolparty][:haproxy_name] %>
-    mode              <%= @node[:poolparty][:proxy_mode] %>
-    bind 0.0.0.0:<%= @node[:poolparty][:ports_haproxy].join(",0.0.0.0:") %>
-    cookie POOLPARTYPARTY
-    <% %x[/usr/bin/server-list-active ip].split("\t").each_with_index do |ip, index| %>
-    server <%= index == 0 ? "master" : "node#{index}" %> <%= ip %>:<%= @node[:poolparty][:forwarding_port] %> weight 1 check cookie
-    <% end %>
+  mode          <%= @node[:poolparty][:proxy_mode] %>
+  bind          0.0.0.0:<%= @node[:poolparty][:ports_haproxy].join(",0.0.0.0:") %>
+  cookie        POOLPARTY
+<% %x[/usr/bin/server-list-active internal_ip].split("\t").each_with_index do |ip, index| %>
+  server        <%= index == 0 ? "master" : "node#{index}" %> <%= ip %>:<%= @node[:poolparty][:forwarding_port] %> weight 1 check cookie
+<% end %>

data/lib/poolparty/templates/monitor.ru

@@ -3,12 +3,12 @@ require 'rubygems'
 require 'rack'
 require 'json'
 require 'thin'
-require "poolparty/monitors"
+require 'rest_client'
+require "poolparty/monitors/monitor_rack" 
 
 app = Rack::Builder.new do
-  use Rack::Reloader, 2
+  # use Rack::Reloader, 2
   use Rack::ShowExceptions
-  use Rack::Lint
   # use Rack::PostBodyContentTypeParser  #parses json requests to params hash
   run Monitors::MonitorRack.new()
 end

data/lib/poolparty/verification/verifier_base.rb

@@ -0,0 +1,17 @@
+module PoolParty
+  module Verifiers
+    
+    class VerifierBase
+      attr_reader :host
+      
+      def host=(h=nil)
+        @host ||= h
+      end
+      
+      def name
+        @name ||= self.class.to_s.top_level_class
+      end
+    end
+    
+  end
+end

data/lib/poolparty/verification/verifiers/ping.rb

@@ -0,0 +1,17 @@
+module PoolParty
+  module Verifiers
+    
+    class Ping < VerifierBase
+      include ::PoolParty::Pinger
+      
+      attr_reader :port
+      def initialize(port=80)
+        @port = port
+      end
+      def passing?
+        ping_port(host, port, 3)
+      end
+    end
+    
+  end
+end