poolparty 0.2.18 → 0.2.69

data/lib/poolparty/pool/base.rb

@@ -10,22 +10,28 @@ module PoolParty
     extend MethodMissingSugar
     
     default_options({
-      :environment => "production",
       :user => "root", # This should change here
-      :base_keypair_path => "~/.ec2",
+      :base_keypair_path => "#{ENV["HOME"]}/.ec2",
       :tmp_path => "/tmp/poolparty",
-      :remote_storage_path => "/var/poolparty",      
-      :fileserver_base => "puppet:///files",
+      :remote_storage_path => "/var/poolparty",
+      :remote_gem_path => "/var/poolparty/gems",
+      :fileserver_base => "puppet://master/files",
       :base_config_directory => "/etc/poolparty",
       :template_directory => "templates",
       :template_path => "/var/lib/puppet/templates",
       :module_path => "/etc/puppet/modules/poolparty",
       :default_specfile_name => "pool.spec",
+      :default_project_specfile_name => "spec/pool.spec",
       :port => "80",
       :forwarding_port => "8080",
-      :proxy_mode => "http",      
+      :proxy_mode => "http",
+      :messenger_client_port => 7050,
       # EC2 Options
-      :ami => "ami-1cd73375" 
+      :ami => "ami-1cd73375",
+      :size => 'm1.small', # must be 'm1.small', 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
+      :security_group => ["default"],
+      # Options that should not be touched pretty much ever
+      :manifest_path => "/etc/puppet/manifests"
     })
         
     # Class methods
@@ -35,13 +41,16 @@ module PoolParty
       end
       # Get the access_key
       def access_key
-        ENV["AWS_ACCESS_KEY_ID"] ? ENV["AWS_ACCESS_KEY_ID"] : load_keys_from_file[:access_key]
+        @access_key ||= ENV["AWS_ACCESS_KEY"] ? ENV["AWS_ACCESS_KEY"] : load_keys_from_file[:access_key]
       end
       def secret_access_key
-        ENV["AWS_SECRET_ACCESS_ID"] ? ENV["AWS_SECRET_ACCESS_ID"] : load_keys_from_file[:secret_access_key]
+        @secret_access_key ||= ENV["AWS_SECRET_ACCESS_KEY"] ? ENV["AWS_SECRET_ACCESS_KEY"] : load_keys_from_file[:secret_access_key]
+      end
+      def read_keyfile
+        open(get_working_key_file_locations).read
       end
       def load_keys_from_file
-        @keys ||= get_working_key_file_locations ? YAML::load( open(get_working_key_file_locations).read ) : {}
+        @keys ||= get_working_key_file_locations ? YAML::load( read_keyfile ) : {}
       end
       # Store the keys in a yaml format to give the master access
       # So that the master has access to the files
@@ -50,6 +59,13 @@ module PoolParty
           write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
         end
       end
+      def store_keys_in_file_for(obj=nil)
+        if obj
+          @access_key = obj.access_key
+          @secret_access_key = obj.secret_access_key
+        end
+        store_keys_in_file
+      end
       def reset!
         @keys = nil
       end
@@ -68,24 +84,37 @@ module PoolParty
           "ppkeys"
         ]
       end
-            
       def storage_directory
         [
             "/var/poolparty"           
         ].select do |dir|
-          dir if ::File.directory?(dir) && ::File.readable?(dir)
+          dir if viable_directory?(dir)
         end.first || ::File.join( "/tmp/poolparty")
       end
-      
+      def logger_location
+        [
+            "/var/log/poolparty"
+        ].select do |dir|
+          dir if viable_directory?(dir)
+        end.first || ::File.join(Dir.pwd, "log")
+      end
+      # Assume the logs will be at the pool.log location within the 
+      # logger_location set above
       def pool_logger_location
-        File.join(Dir.pwd, "logs")
+        ::File.join(logger_location, "pool.log")
+      end
+      def custom_monitor_directories
+        [
+          "/var/poolparty/monitors",
+          "/etc/poolparty/monitors",
+          "#{Dir.pwd}/monitors"
+        ].select {|d| d if viable_directory?(d) }
       end
-      
-      # Array of allowed_commands that you can run on the remote nodes
-      def allowed_commands
-        @allowed_commands ||= open(::File.join( ::File.dirname(__FILE__), "..", "config", "allowed_commands.yml")).read.split(/\n/).map {|a| a.chomp }
+      # Only return true if the directory we are reading is both readable
+      # and exists
+      def viable_directory?(dir)
+        ::File.directory?(dir) && ::File.readable?(dir)
       end
-      
     end
   end    
 end

data/lib/poolparty/pool/cloud.rb

@@ -3,8 +3,8 @@ require File.dirname(__FILE__) + "/resource"
 
 module PoolParty    
   module Cloud
-    def cloud(name=:main, &block)
-      clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, self, &block))
+    def cloud(name=:app, parent=self, &block)
+      clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, parent, &block))
     end
 
     def clouds
@@ -12,56 +12,69 @@ module PoolParty
     end
     
     def with_cloud(cl, opts={}, &block)
-      cl.options.merge!(opts)
-      cl.instance_eval &block if block
+      raise CloudNotFoundException.new("Cloud not found") unless cl
+      cl.options.merge!(opts) if opts
+      cl.run_in_context &block if block
     end
     
     class Cloud
-      attr_reader :name, :templates
+      attr_reader :templates
       include PoolParty::PluginModel
       include PoolParty::Resources      
       include PrettyPrinter
       include Configurable
       include CloudResourcer
       # extend CloudResourcer
-      # Net methods
-      include PoolParty::Remote::RemoterBase
+      # Net methods      
       include Remote
+      include PoolParty::CloudDsl
       
       default_options({
         :minimum_instances => 2,
-        :maximum_instances => 4,
+        :maximum_instances => 5,
+        :contract_when => "cpu < 0.65",
+        :expand_when => "cpu > 1.9",
         :access_key => Base.access_key,
         :secret_access_key => Base.secret_access_key,
         :ec2_dir => ENV["EC2_HOME"],
         :keypair => (ENV["KEYPAIR_NAME"].nil? || ENV["KEYPAIR_NAME"].empty?) ? nil : ENV["KEYPAIR_NAME"],
-        :ami => 'ami-44bd592d',
-        :polling_time => "30.seconds"
+        :ami => 'ami-44bd592d'
       })
       
-      def initialize(name, parent, &block)
-        @name = name
-        # store_block(&block)
-        set_parent(parent) if parent
-        self.instance_eval &block if block
+      def initialize(name, pare=self, &block)
+        @cloud_name = name
+        @cloud_name.freeze
+                
+        plugin_directory "#{::Dir.pwd}/plugins"
+                
+        p = pare.is_a?(PoolParty::Pool::Pool) ? pare : nil
+        run_setup(p, &block)
+        
+        # set_parent(parent) if parent && !@parent
+        # self.run_in_context parent, &block if block
+        setup_defaults
+        reset_remoter_base!        
+      end
+      
+      def setup_defaults
         # this can be overridden in the spec, but ec2 is the default
         self.using :ec2
+        generate_keypair unless has_keypair?
       end
-                              
-      # Keypairs
-      # If the parent (pool) doesn't have a keypair defined on it, then generate one based on the 
-      # pool_name and the cloud_name
-      def keypair(*args)
-        has_keypair? ? options[:keypair] : generate_keypair(*args)
+      
+      def name
+        @cloud_name
       end
+                                    
+      # Keypairs
       # Let's just make sure that the keypair exists on the options
       def has_keypair?
-        options.has_key?(:keypair) && options[:keypair]
+        options.has_key?(:keypair) && options[:keypair] && !options[:keypair].empty?
       end
       # Generate a keypair based on the parent's name (if there is a parent)
       # and the cloud's name
       def generate_keypair(*args)
-        options[:keypair] = args.length > 0 ? args[0] : "#{@parent && @parent.respond_to?(:name) ? @parent.name : ""}_#{@name}"
+        options[:keypair] = "#{parent && parent.is_a?(PoolParty::Pool::Pool) ? parent.name : "poolparty"}_#{name}" unless has_keypair?
       end
       
       # Prepare to send the new configuration to the instances
@@ -72,19 +85,29 @@ module PoolParty
       # Then, send the saved containing cloud instances to give the 
       # remote master access to the cloud options that are required
       # for the master to run checks
-      def prepare_to_configuration        
+      def prepare_for_configuration        
         # clear_base_directory
         make_base_directory
         copy_misc_templates
-        Base.store_keys_in_file
-        Script.save!
-        copy_ssh_key # not my favorite...
+        copy_custom_monitors
+        store_keys_in_file
+        Script.save!(self)
+        # not my favorite...
+        copy_ssh_key
       end
       
+      # Copy the ssh keys to the storage directory in preparation for
+      # configuration
       def copy_ssh_key
         copy_file_to_storage_directory(full_keypair_path)
       end
       
+      # Store our keys for cloud access in a file 
+      # that is specific to this cloud
+      def store_keys_in_file
+        Base.store_keys_in_file_for(self)
+      end
+      
       # Build the new poolparty manifest
       # Wrapping all of these requirements into the one 
       # poolparty class.
@@ -92,61 +115,92 @@ module PoolParty
       # TODO: Consider the benefits of moving all the manifest
       # classes to separate files and keeping the containing
       # references in the include
-      def build_and_store_new_config_file
-        @manifest = build_manifest
+      def build_and_store_new_config_file(force=false)
+        vputs "Building new manifest configuration file (forced: #{force})"
+        manifest = force ? rebuild_manifest : build_manifest
         config_file = ::File.join(Base.storage_directory, "poolparty.pp")
-        ::File.open(config_file, "w+") do |file|
-          file << "class poolparty {"
-          file << @manifest
-          file << "}"
+        ::File.open(config_file, "w") do |file|
+          file << manifest
         end
       end      
       
       def copy_misc_templates
-        ["fileserver.conf", "namespaceauth.conf"].each do |f|
+        ["namespaceauth.conf"].each do |f|
           copy_file_to_storage_directory(::File.join(::File.dirname(__FILE__), "..", "templates", f))
         end
       end
+      
+      # If there is a directory named monitors in the same directory
+      # as the pool specification file is in,
+      # then create a monitors directory in the storage directory
+      # and mirror the two. When PoolParty "boots" up, it scans
+      # the monitors directory for any custom monitors
+      # that are in known locations, these are included
+      def copy_custom_monitors
+        unless Base.custom_monitor_directories.empty?
+          make_directory_in_storage_directory("monitors")
+          Dir["#{Base.custom_monitor_directories}/*.rb"].each do |f|
+            copy_file_to_storage_directory(f, "monitors")
+          end
+        end        
+      end
             
       # Configuration files
       def build_manifest
+        vputs "Building manifest"
         @build_manifest ||= build_from_existing_file
         unless @build_manifest
-          reset_resources!
+          
           add_poolparty_base_requirements
           
-          @build_manifest = returning Array.new do |str|
-
-            str << resources_string_from_resources(resources)
-
-            # Refactor this into the resources method
-            # TODO
-            services.each do |service|
-              @cp = classpackage_with_self(service)
-              str << @cp.to_string
-              str << @cp.include_string
-            end
-
-            str << "# Custom functions"
-            str << Resources::CustomResource.custom_functions_to_string
-          end.join("\n")          
+          @build_manifest = "class poolparty {\n #{build_short_manifest}\n}"
         end
         @build_manifest
       end
       
+      def rebuild_manifest
+        @build_manifest = nil
+        build_manifest
+      end
+      
+      def build_short_manifest
+        returning Array.new do |str|            
+
+          # Refactor this into the resources method
+          # TODO
+          services.each do |service|
+            service.options.merge!(:name => service.name)
+            classpackage_with_self(service)
+          end
+          
+          options.merge!(:name => "user")
+          classpackage_with_self
+          # resources.each do |type, res|
+          #   str << "# #{type.to_s.pluralize}"
+          #   str << res.to_string
+          # end
+          
+          global_classpackages.each do |cls|
+            str << cls.to_string
+          end
+
+          str << "# Custom functions"
+          str << Resources::CustomResource.custom_functions_to_string
+        end.join("\n")
+      end
+      
       def build_from_existing_file
-        if ::FileTest.file?("/etc/puppet/manifests/classes/poolparty.pp")
-          open("/etc/puppet/manifests/classes/poolparty.pp").read
-        else
-          nil
-        end
+        ::FileTest.file?("#{Base.manifest_path}/classes/poolparty.pp") ? open("#{Base.manifest_path}/classes/poolparty.pp").read : nil
       end
       
       # To allow the remote instances to do their job,
       # they need a few options to run, these are the required options
       # to be saved on the remote "master" machine
       def minimum_runnable_options
-        [:keypair, :minimum_instances, :maximum_instances]
+        [
+          :keypair, :minimum_instances, :maximum_instances, :ami, :security_group,
+          :expand_when, :contract_when, :set_master_ip_to
+        ]
       end
       
       # Add all the poolparty requirements here
@@ -159,19 +213,21 @@ module PoolParty
         heartbeat
         haproxy
         ruby
-        poolparty
+        poolparty_base_packages
+        realize_plugins!
       end
       
-      def provisioning?
-        @provisioning_in_progress == true
-      end
-      
-      def provisioning_in_progress
-        @provisioning_in_progress = true
+      def other_clouds
+        returning Array.new do |arr|
+          clouds.each do |name, cl|
+            arr << cl if name != self.name
+          end
+        end
       end
       
-      def provisioning_complete
-        @provisioning_in_progress = false
+      def reset!
+        reset_remoter_base!
+        @build_manifest = @describe_instances = nil
       end
             
       # Add to the services pool for the manifest listing

data/lib/poolparty/pool/custom_resource.rb

@@ -4,42 +4,26 @@ module PoolParty
   def available_custom_resources
     $available_custom_resources ||= []
   end
-  module DefinableFact
-    def define_fact(name, string="")
-      
-    end
-  end
 
   module Resources
     
-    def call_function(str, opts={}, &block)
-      returning PoolParty::Resources::CallFunction.new(str, opts, &block) do |o|
-        resource(:call_function) << o
-      end
+    def call_custom_function(str, opts={}, parent=self, &block)
+      add_resource(:call_function, opts.merge({:str => str, :name => str.keyerize}), parent, &block)
     end
                 
     # Resources for function call
     class CallFunction < Resource
-      def initialize(str="", opts={}, parent=self, &block)
-        @str = str
-        # super(opts, parent, &block)
-      end
       def to_string(pre="")
         returning Array.new do |arr|
-          arr << "#{pre}#{@str}"
+          arr << "#{pre}#{str}"
         end.join("\n")
       end
     end
         
     class CustomResource < Resource
-      def initialize(name=:custom_method, opts={}, parent=self, &block)
-        @name = name
-        super(opts, parent, &block)
-      end
-      
       def self.inherited(subclass)
         PoolParty::Resources.available_custom_resources << subclass
-        super(subclass)
+        super
       end
       
       def to_string(pre="")

data/lib/poolparty/pool/loggable.rb

@@ -2,17 +2,12 @@
   A logging class to allow us to log to locations
 =end
 class Loggable
-  def initialize
-    Logging.init :debug, :info, :warn, :error, :fatal
-    
+  def initialize    
     self.class.loggers << file_logger
-    file_logger.level = :warn
-
-    self.class.loggers << stdout_logger
-    stdout_logger.level = :info
+    file_logger.level = :info
   end
   def file_logger
-    @file_logger ||= Logging.logger( ::File.join(Base.pool_logger_location, "pool_log.log"), logging_opts )
+    @file_logger ||= Logging.logger( Base.pool_logger_location, logging_opts )
   end
   def stdout_logger
     @stdout_logger ||= Logging.logger(STDOUT, logging_opts.merge({:pattern => "%m\n"}))

data/lib/poolparty/pool/plugin.rb

@@ -1,39 +1,34 @@
 module PoolParty
       
   module Plugin
-    
+        
     class Plugin
       include Configurable
       include CloudResourcer
       include Resources
       
-      attr_accessor :parent
       class_inheritable_accessor :name
       
       default_options({})
       
-      def initialize(parent=self, opts={}, &block)
-        set_parent(parent)
-        block ? instance_eval(&block) : enable
+      def initialize(p=self, opts={}, &block)
+        store_block &block
+        run_setup(p)
+        realize! unless block
+      end
+            
+      def realize!(force=false)
+        force ? force_realize! : (@realized ? nil : force_realize!)
+      end
+      
+      def force_realize!
+        run_setup(parent, false, &stored_block)
+        enable unless stored_block
       end
       
       # Overwrite this method
       def enable
       end
-            
-      # def method_missing(m, *args, &block)
-      #   if block_given?
-      #     (args[0].class == self.class) ? args[0].instance_eval(&block) : super
-      #   elsif parent && parent.respond_to?(m)
-      #     parent.send m, *args, &block
-      #   else
-      #     get_from_options(m, *args)
-      #   end
-      # end
-      # 
-      # def get_from_options(m, *args)
-      #   args.empty? ? options[m] : options[m] = args[0]
-      # end
       
     end
     

data/lib/poolparty/pool/plugin_model.rb

@@ -14,30 +14,30 @@ module PoolParty
     
     class PluginModel      
       attr_accessor :name, :klass
-      attr_reader :parent
       include MethodMissingSugar
       include Configurable
       include PrettyPrinter      
       
       def initialize(name,cld,&block)
         @name = name
-        @parent = cld
+        # @parent = cld
         class_string_name = "#{name}"
         
         # Create the class to evaluate the plugin on the implemented call
         klass = class_string_name.class_constant(PoolParty::Plugin::Plugin)
-        klass.extend PoolParty::Resources
-        klass.send :include, PoolParty::Resources
+        mod = class_string_name.module_constant(&block)
 
-        # Create the block inside the instantiated plugin
-        klass.module_eval &block if block
+        klass.send :include, mod
         
         # Store the name of the class for pretty printing later
         klass.name = name
         # Add the plugin definition to the cloud as an instance method
-        Cloud::Cloud.module_eval <<-EOE
+        PoolParty::Cloud::Cloud.class_eval <<-EOE
           def #{name}(parent=self, &block)
-            @#{class_string_name.downcase} ||= #{class_string_name.class_constant}.new(parent, &block)
+            @pa = parent
+            @#{class_string_name.downcase} ||= returning #{class_string_name.class_constant}.new(parent, &block) do |pl|
+              @pa.plugin_store << pl
+            end
           end
         EOE
       end

data/lib/poolparty/pool/pool.rb

@@ -1,7 +1,7 @@
 module PoolParty
   module Pool
     
-    def pool(name=:main, &block)
+    def pool(name=:app, &block)
       pools.has_key?(name) ? pools[name] : (pools[name] = Pool.new(name, &block))
     end    
     
@@ -9,18 +9,18 @@ module PoolParty
       $pools ||= {}
     end
     
-    def with_pool(pool, opts={}, &block)
-      pool.options.merge!(opts)
-      pool.instance_eval &block if block
+    def with_pool(pl, opts={}, &block)
+      raise CloudNotFoundException.new("Pool not found") unless pl
+      pl.options.merge!(opts) if pl.options
+      pl.run_in_context &block if block
     end
         
     def reset!
-      $pools = $clouds = $plugins = nil
+      $pools = $clouds = $plugins = @describe_instances = nil
     end
 
     class Pool
-      attr_accessor :name
-      include PoolParty::Cloud
+      # include PoolParty::Cloud
       include MethodMissingSugar
       # include PluginModel
       include Configurable
@@ -34,17 +34,34 @@ module PoolParty
       })
       
       def initialize(name,&block)
-        @name = name
-        instance_eval &block if block
+        setup_defaults
+        
+        @pool_name = name
+        @pool_name.freeze
+        # run_in_context &block if block
+        run_setup(self, &block)
+      end
+      
+      def name
+        @pool_name
       end
       
-      def plugin_directory(*args)
-        args.each {|arg| Dir["#{arg}/*/*.rb"].each {|f| require f }}
+      def setup_defaults
+        plugin_directory "#{::File.dirname(pool_specfile ? pool_specfile : Dir.pwd)}/plugins"
       end
             
       # This is where the entire process starts
       def inflate
       end
+      
+      def pool_clouds
+        returning Array.new do |arr|
+          clouds.each do |name, cl|
+            arr << cl if cl.parent.name == self.name
+          end
+        end
+      end
+      
     end
     
     # Helpers