poolparty 0.2.84 → 1.2.2

data/lib/poolparty/plugins/svn.rb

@@ -1,47 +1,64 @@
 module PoolParty    
-  class Svn
+  class SvnResource
         
-    define_resource(:svn) do
+    virtual_resource :svn do
+      def loaded(*args)
+        has_package :name => "subversion"
+      end
+    end
+    
+    virtual_resource :svn_repos do
+      
+      def loaded(opts={}, &block)
+        has_package("subversion")
+        has_svn_repository
+      end
+
+      def has_svn_repository
+        puts "wd #{working_dir}"
+        puts "cd #{creates_dir}"
+        has_directory(::File.dirname(working_dir))
+        has_directory(:name => "#{working_dir}", :requires => get_directory("#{::File.dirname(working_dir)}"))
+        
+        has_exec(:name => "svn-#{name}", :requires => [get_directory("#{working_dir}"), get_package("subversion")] ) do
+          cwd working_dir
+          svn_cmd = if parent.requires_user?
+            "svn co #{source} --username #{requires_user} --password #{requires_password}" 
+          else
+            "svn co #{source}"
+          end
+          command svn_cmd
+          creates creates_dir
+        end
+        has_exec(:name => "update-#{name}", :cwd => ::File.dirname( creates_dir )) do
+          command "svn up"
+          # If the parent has after_update_svn set on it, then run it
+          runs parent.after_update_svn if parent.after_update_svn?
+        end
+      end
+      
+      def svn_repos(src)
+        source src
+      end
+      
+      def at(dir)
+        working_dir dir
+      end
+      
+      def to(dir)
+        at(dir)
+      end
+      
+      def creates_dir
+       "#{::File.join( working_dir, ::File.basename(source, ::File.extname(source)) )}/.svn"
+      end
       
-      def has_svnpath(opts={})
-        call_custom_function <<-EOE
-        svnserve { #{opts[:name]}:
-          source => "#{opts[:source]}",
-          path => "#{opts[:path]}",
-          user => "#{opts[:user] || false}",
-          password => "#{opts[:password] || ""}"
-        }
-        EOE
+      # Since svn is not a native type, we have to say which core resource
+      # it is using to be able to require it
+      def class_type_name
+        "exec"
       end
       
-      custom_function <<-EOF
-      # Serve subversion-based code from a local location.  The job of this
-      # module is to check the data out from subversion and keep it up to
-      # date, especially useful for providing data to your Puppet server.
-      define svnserve($source, $path, $user = false, $password = false) {
-          file { $path:
-              ensure => directory,
-              owner => root,
-              group => root
-          }
-          $svncmd = $user ? {
-              false => "/usr/bin/svn co --non-interactive $source/$name .",
-              default => "/usr/bin/svn co --non-interactive --username $user --password '$password' $source/$name ."
-          }   
-          exec { "svnco-$name":
-              command => $svncmd,
-              cwd => $path,
-              require => File[$path],
-              creates => "$path/.svn"
-          }
-          exec { "svnupdate-$name":
-              command => "/usr/bin/svn update",
-              require => Exec["svnco-$name"],
-              onlyif => '/usr/bin/svn status -u --non-interactive | /bin/grep "\*"',
-              cwd => $path
-          }
-      }
-      EOF
     end
     
   end

data/lib/poolparty/poolparty/cloud.rb

@@ -3,133 +3,143 @@ require File.dirname(__FILE__) + "/resource"
 
 module PoolParty    
   module Cloud
-    def cloud(name=:app, parent=self, &block)
-      clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, parent, &block))
+    # Instantiate a new cloud
+    def cloud(name, &block)
+      clouds[name] ||= Cloud.new(name, &block)
     end
 
+    # Global hash of clouds
     def clouds
       $clouds ||= {}
     end
     
+    # TODO: Deprecate
     def with_cloud(cl, opts={}, &block)
       raise CloudNotFoundException.new("Cloud not found") unless cl
       cl.options.merge!(opts) if opts
       cl.run_in_context &block if block
     end
     
-    class Cloud
-      attr_reader :templates
+    class Cloud < PoolParty::PoolPartyBaseClass
+      attr_reader :templates, :cloud_name, :remote_base
+
+      include CloudResourcer
       include PoolParty::PluginModel
-      include PoolParty::Resources      
+      include PoolParty::Resources
+      include PoolParty::Callbacks
+      include PoolParty::DependencyResolverCloudExtensions
       include PrettyPrinter
-      include Configurable
-      include CloudResourcer
-      include Provisioner
-      # extend CloudResourcer
-      # Net methods      
-      include Remote
+
+      # Net methods
+      include ::PoolParty::Remote
       include PoolParty::CloudDsl
+      include PoolParty::Verification
+      # include PoolParty::Monitors
+
+      def verbose
+        true
+      end
+      
+      def self.immutable_methods
+        [:name]
+      end
+      
+      # Redefining methods are not allowed
+      def self.method_added sym        
+        raise "Exception: #{sym.to_s.capitalize} method has been redefined" if immutable_methods.include?(sym) && !respond_to?(sym)
+      end
+      
+      alias :name :cloud_name
+      
+      # Call the remoter commands on the cloud if they don't exist on the cloud itself
+      # This gives the cloud access to the remote_base's methods
+      def method_missing(m, *args, &block)
+        remote_base.respond_to?(m) ? remote_base.send(m, *args, &block) : super
+      end
       
-      default_options({
+      # Default set of options. Most take the Default options from the default class
+      default_options(
+        :expand_when => Default.expand_when,
+        :contract_when => Default.contract_when,
         :minimum_instances => 2,
         :maximum_instances => 5,
-        :contract_when => "cpu < 0.65",
-        :expand_when => "cpu > 1.9",
-        :access_key => Base.access_key,
-        :secret_access_key => Base.secret_access_key,
+        :access_key => Default.access_key,
+        :secret_access_key => Default.secret_access_key,
         :ec2_dir => ENV["EC2_HOME"],
-        :keypair => (ENV["KEYPAIR_NAME"].nil? || ENV["KEYPAIR_NAME"].empty?) ? nil : ENV["KEYPAIR_NAME"],
-        :minimum_runtime => Base.minimum_runtime,
-        :user => Base.user,
-        :ami => 'ami-44bd592d'
-      })
+        :minimum_runtime => Default.minimum_runtime,
+        :user => Default.user
+      )
       
-      def initialize(name, pare=self, &block)
+      additional_callbacks [
+        "after_launch_instance"
+      ]
+      
+      # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
+      # call and run instance_eval on the block and then call the after_create callback
+      def initialize(name, &block)
         @cloud_name = name
         @cloud_name.freeze
-                
-        plugin_directory
-                
-        p = pare.is_a?(PoolParty::Pool::Pool) ? pare : nil
-        store_block(&block)
-        run_setup(p, &block)        
         
-        # set_parent(parent) if parent && !@parent
-        # self.run_in_context parent, &block if block
-        setup_defaults
-        # realize_plugins!
-        # reset! # reset the clouds
-        # reset_remoter_base!
+        plugin_directory "#{pool_specfile ? ::File.dirname(pool_specfile) : Dir.pwd}/plugins"        
+        before_create
+        super
+        after_create
       end
       
-      def setup_defaults
-        # this can be overridden in the spec, but ec2 is the default
-        self.using :ec2
-        generate_keypair unless has_keypair?
+      # Fetch the name of the cloud
+      def name(*args)
+        @cloud_name ||= @cloud_name ? @cloud_name : (args.empty? ? :default_cloud : args.first)
       end
       
-      def name
-        @cloud_name
-      end
-                                    
-      # Keypairs
-      # Let's just make sure that the keypair exists on the options
-      def has_keypair?
-        options.has_key?(:keypair) && options[:keypair] && !options[:keypair].empty?
-      end
-      # Generate a keypair based on the parent's name (if there is a parent)
-      # and the cloud's name
-      def generate_keypair(*args)
-        options[:keypair] = "#{parent && parent.is_a?(PoolParty::Pool::Pool) ? parent.name : "poolparty"}_#{name}" unless has_keypair?
-      end
-            
-      # Prepare to send the new configuration to the instances
-      # First, let's make sure that our base directory is made
-      # Then copy the templates that have no other reference in
-      # a spec file. Make sure the keys are stored in a file
-      # For the master to have access to them
-      # Then, send the saved containing cloud instances to give the 
-      # remote master access to the cloud options that are required
-      # for the master to run checks
-      def prepare_for_configuration        
-        # clear_base_directory
-        make_base_directory
-        copy_misc_templates
-        copy_custom_monitors
-        store_keys_in_file
-        Script.save!(self)
-        # not my favorite...
-        copy_ssh_key
-        write_unique_cookie
-        before_configuration_tasks
+      def before_create     
+        context_stack.push self
+        (parent ? parent : self).add_poolparty_base_requirements
+        context_stack.pop
       end
       
-      # Copy the ssh keys to the storage directory in preparation for
-      # configuration
-      def copy_ssh_key
-        copy_file_to_storage_directory(full_keypair_path)
+      # Callback
+      # called after the cloud has been created, everything has run and is set at this point
+      # here the base requirements are added as well as an empty chef recipe is called
+      # Also, the after_create hook on the plugins used by the cloud are called here
+      def after_create
+        ::FileUtils.mkdir_p("#{Default.tmp_path}/dr_configure")
+        
+        run_in_context do
+          add_optional_enabled_services
+          chef do
+          end
+        end
+        
+        plugin_store.each {|a| a.call_after_create_callbacks }
+        setup_defaults
+        
+        setup_callbacks
       end
       
-      # Store our keys for cloud access in a file 
-      # that is specific to this cloud
-      def store_keys_in_file
-        Base.store_keys_in_file_for(self)
+      # setup defaults for the cloud
+      def setup_defaults
+        # this can be overridden in the spec, but ec2 is the default
+        using :ec2
+        options[:keypair] ||= keypair rescue nil
+        options[:rules] = {:expand => dsl_options[:expand_when], :contract => dsl_options[:contract_when]}
+        dependency_resolver 'chef'        
+        # enable :haproxy unless dsl_options[:haproxy] == :disabled
       end
       
-      # Let's write the cookie into the tmp path
-      def write_unique_cookie
-        write_to_file_in_storage_directory("cookie") do
-          generate_unique_cookie_string
-        end
+      def after_launch_instance(inst=nil)
+        remote_base.send :after_launch_instance, inst
       end
       
-      # Generate a unique cookie string so that our erlang modules can 
-      # talk to each other safely. This is based off the keypair
-      # and the name of the cloud
-      def generate_unique_cookie_string
-        Digest::SHA256.hexdigest("#{full_keypair_name}#{name}")
+      # provide list of public ips to get into the cloud
+      def ips
+        nodes(:status => "running").map {|ri| ri.ip }
       end
       
+      # TODO: make this be a random ip, since we should not rely on it being the same each time
+      def ip
+        ips.first
+      end
+            
       # Build the new poolparty manifest
       # Wrapping all of these requirements into the one 
       # poolparty class.
@@ -137,19 +147,14 @@ module PoolParty
       # TODO: Consider the benefits of moving all the manifest
       # classes to separate files and keeping the containing
       # references in the include
-      def build_and_store_new_config_file(force=false)
+      def build_and_store_new_config_file(filepath=nil, force=false)
+        filepath ||= ::File.join(Default.storage_directory, "poolparty.pp")
+        # write_properties_hash if debugging
         vputs "Building new manifest configuration file (forced: #{force})"
         manifest = force ? rebuild_manifest : build_manifest
-        config_file = ::File.join(Base.storage_directory, "poolparty.pp")
-        ::File.open(config_file, "w") do |file|
+        ::File.open(filepath, "w") do |file|
           file << manifest
         end
-      end      
-      
-      def copy_misc_templates
-        ["namespaceauth.conf", "puppet.conf", "gem"].each do |f|
-          copy_file_to_storage_directory(::File.join(::File.dirname(__FILE__), "..", "templates", f))
-        end
       end
       
       # If there is a directory named monitors in the same directory
@@ -159,70 +164,45 @@ module PoolParty
       # the monitors directory for any custom monitors
       # that are in known locations, these are included
       def copy_custom_monitors
-        unless Base.custom_monitor_directories.empty?
-          make_directory_in_storage_directory("monitors")
-          Dir["#{Base.custom_monitor_directories}/*.rb"].each do |f|
-            copy_file_to_storage_directory(f, "monitors")
+        unless Default.custom_monitor_directories.empty?
+          Default.custom_monitor_directories.each do |dir|
+            Dir["#{dir}/*.rb"].each {|f| ::Suitcase::Zipper.add(f, "monitors")}
           end
-        end        
+        end
       end
-            
+
+      #FIXME MOVE TO DEPENDECY RESOL
       # Configuration files
       def build_manifest
         vputs "Building manifest"
         @build_manifest ||= build_from_existing_file
-        unless @build_manifest
-          
-          add_poolparty_base_requirements
+        unless @build_manifest          
+          props = to_properties_hash
           
-          @build_manifest = "class poolparty {\n #{build_short_manifest}\n}"
+          @build_manifest =  options[:dependency_resolver].send(:compile, props, self)
         end
+        dputs "Finished creating manifest"
         @build_manifest
       end
       
+      # Force rebuilding of the manifest
       def rebuild_manifest
         @build_manifest = nil
         build_manifest
       end
       
-      def build_short_manifest
-        returning Array.new do |str|            
-
-          # Refactor this into the resources method
-          # TODO
-          services.each do |service|
-            service.options.merge!(:name => service.name)
-            classpackage_with_self(service)
-          end
-          
-          options.merge!(:name => "user")
-          classpackage_with_self
-          # resources.each do |type, res|
-          #   str << "# #{type.to_s.pluralize}"
-          #   str << res.to_string
-          # end
-          
-          global_classpackages.each do |cls|
-            str << cls.to_string
-          end
-
-          str << "# Custom functions"
-          str << Resources::CustomResource.custom_functions_to_string
-        end.join("\n")
-      end
-      
+      # If the pp already exists, then let's not recreate it
+      # TODO: Abstract
       def build_from_existing_file
-        ::FileTest.file?("#{Base.manifest_path}/classes/poolparty.pp") ? open("#{Base.manifest_path}/classes/poolparty.pp").read : nil
+        ::FileTest.file?("#{Default.base_config_directory}/poolparty.pp") ? open("#{Default.base_config_directory}/poolparty.pp").read : nil
       end
       
-      # To allow the remote instances to do their job,
-      # they need a few options to run, these are the required options
-      # to be saved on the remote "master" machine
-      def minimum_runnable_options
-        ([
-          :keypair, :minimum_instances, :maximum_instances,
-          :expand_when, :contract_when, :set_master_ip_to
-        ]<< custom_minimum_runnable_options).flatten
+      def write_properties_hash(filename=::File.join(Default.tmp_path, Default.properties_hash_filename) )
+        file_path = ::File.dirname(filename)
+        file_name = "#{::File.basename(filename, ::File.extname(filename))}_#{name}#{::File.extname(filename)}"
+        output = to_properties_hash.to_json
+        ::File.open("#{file_path}/#{file_name}", "w") {|f| f.write output }
+        true
       end
       
       # Add all the poolparty requirements here
@@ -232,35 +212,25 @@ module PoolParty
       # all that is necessary in a method called enable
       # which is called when there is no block
       def add_poolparty_base_requirements
-        heartbeat
-        haproxy
-        ruby
-        poolparty_base_packages
-        realize_plugins!(true) # Force realizing of the plugins
+        # poolparty_base_heartbeat
+        poolparty_base_ruby
+        poolparty_base_packages        
       end
       
+      # TODO: Deprecate
       def other_clouds
-        returning Array.new do |arr|
-          clouds.each do |name, cl|
-            arr << cl if name != self.name
-          end
+        arr = []
+        clouds.each do |name, cl|
+          arr << cl if name != self.name
         end
+        arr
       end
       
+      # Reset the entire cloud
       def reset!
         reset_remoter_base!
-        @build_manifest = @describe_instances = nil
-      end
-            
-      # Add to the services pool for the manifest listing
-      def add_service(serv)
-        services << serv
-      end
-      # Container for the services
-      def services
-        @services ||= []
-      end
-            
+        @build_manifest = @describe_instances = @remote_instances_list = nil
+      end            
     end
-  end  
-end
+  end 
+end

data/lib/poolparty/poolparty/{base.rb → default.rb}

@@ -5,14 +5,20 @@
 require "open-uri"
 require "ftools"
 module PoolParty
-  class Base
-    include Configurable
-    extend MethodMissingSugar
+  class Default
+    include Dslify
     
-    default_options({
+    # def self.options
+    default_options(
+      :testing => false,
+      :debugging => false,
+      :minimum_instances => 2,
+      :maximum_instances => 5,
       :user => "root", # This should change here
       :base_keypair_path => "#{ENV["HOME"]}/.ec2",
+      :base_ssh_path => "#{ENV["HOME"]}/.ssh",
       :tmp_path => "/tmp/poolparty",
+      :poolparty_home_path => "#{ENV["HOME"]}/.poolparty",
       :remote_storage_path => "/var/poolparty",
       :remote_gem_path => "/var/poolparty/gems",
       :fileserver_base => "puppet://master/files",
@@ -20,33 +26,44 @@ module PoolParty
       :template_directory => "templates",
       :template_path => "/var/lib/puppet/templates",
       :module_path => "/etc/puppet/modules/poolparty",
-      :default_specfile_name => "clouds.pool",
-      :default_project_specfile_name => "spec/clouds.pool",
+      :default_specfile_name => "clouds.rb",
+      :properties_hash_filename => "clouds.json",
+      :vendor_path => "#{::File.dirname(__FILE__)}/../../../vendor",
+      :poolparty_src_path => "#{::File.dirname(__FILE__)}/../../..",
       :port => "80",
       :forwarding_port => "8080",
+      :monitor_port => 8081,
       :proxy_mode => "http",
-      :messenger_client_port => 7050,
+      :butterfly_port => 8642,
       :minimum_runtime  => 3000, #50.minutes in seconds
-      # EC2 Options
-      :ami => "ami-1cd73375",
-      :size => 'm1.small', # must be 'm1.small', 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'
-      :availabilty_zone => "us-east-1a",
-      :security_group => ["default"],
-      # Options that should not be touched pretty much ever
-      :manifest_path => "/etc/puppet/manifests"
-    })
-        
+      :contract_when => "load < 0.25",
+      :expand_when => "load > 0.9"
+    )
+    
+    def options
+      default_options
+    end
+    
     # Class methods
     class << self
-      def options(h={})
-        @options ||= default_options.merge(h)
+      def method_missing(m,*a,&block)
+        default_options.include?(m) ? default_options[m] : super
+      end
+      def options
+        default_options
       end
       # Get the access_key
       def access_key
-        @access_key ||= ENV["AWS_ACCESS_KEY"] ? ENV["AWS_ACCESS_KEY"] : load_keys_from_file[:access_key]
+        @access_key ||= load_access_keys_from_environment_var || load_keys_from_file[:access_key]
+      end
+      def load_access_keys_from_environment_var
+        [ ENV["AWS_ACCESS_KEY"], ENV["AWS_ACCESS_KEY_ID"]].reject {|a| a.nil? }.first
       end
       def secret_access_key
-        @secret_access_key ||= ENV["AWS_SECRET_ACCESS_KEY"] ? ENV["AWS_SECRET_ACCESS_KEY"] : load_keys_from_file[:secret_access_key]
+        @secret_access_key ||= load_secret_access_keys_from_environment_var || load_keys_from_file[:secret_access_key]
+      end
+      def load_secret_access_keys_from_environment_var
+        [ ENV["AWS_SECRET_ACCESS_KEY"] ].reject {|a| a.nil? }.first
       end
       def read_keyfile
         open(get_working_key_file_locations).read
@@ -71,6 +88,7 @@ module PoolParty
       def reset!
         @keys = nil
       end
+      
       # Get the instance first instance file that exists on the system from the expected places
       # denoted in the local_instances_list_file_locations
       def get_working_key_file_locations
@@ -80,15 +98,24 @@ module PoolParty
       def key_file_locations
         [
           ".ppkeys",
-          "#{Base.base_config_directory}/.ppkeys",
-          "#{Base.storage_directory}/ppkeys",          
+          "#{Default.base_config_directory}/.ppkeys",
+          "#{Default.storage_directory}/ppkeys",          
           "~/.ppkeys",
           "ppkeys"
         ]
       end
+      def properties_hash_file
+        [
+          Default.base_config_directory,
+          Dir.pwd
+        ].collect do |dir|
+          full_dir = ::File.join(dir, Default.properties_hash_filename)
+          full_dir if ::File.file?(full_dir)
+        end.compact.first || "#{Default.base_config_directory}/#{Default.properties_hash_filename}"
+      end
       def storage_directory
         [
-            "/var/poolparty"           
+            "/var/poolparty"
         ].select do |dir|
           dir if viable_directory?(dir)
         end.first || ::File.join( "/tmp/poolparty")
@@ -98,12 +125,12 @@ module PoolParty
             "/var/log/poolparty"
         ].select do |dir|
           dir if viable_directory?(dir)
-        end.first || ::File.join(Dir.pwd, "log")
+        end.first
       end
       # Assume the logs will be at the pool.log location within the 
       # logger_location set above
       def pool_logger_location
-        ::File.join(logger_location, "pool.log")
+        ::File.join(logger_location, "poolparty.log")
       end
       def custom_monitor_directories
         [
@@ -112,11 +139,21 @@ module PoolParty
           "#{Dir.pwd}/monitors"
         ].select {|d| d if viable_directory?(d) }
       end
+      
+      def custom_modules_directories
+        [
+          "/var/poolparty/modules",
+          "/etc/poolparty/modules",
+          "#{Dir.pwd}/modules"
+        ].select {|d| d if viable_directory?(d) }
+      end
       # Only return true if the directory we are reading is both readable
       # and exists
       def viable_directory?(dir)
         ::File.directory?(dir) && ::File.readable?(dir)
       end
     end
+    #end of class methods
+    
   end    
 end