fairchild-poolparty 1.1.5 → 1.2.12

data/lib/poolparty/poolparty/cloud.rb

@@ -1,4 +1,3 @@
-require File.dirname(__FILE__) + "/plugin_model"
 require File.dirname(__FILE__) + "/resource"
 
 module PoolParty    
@@ -16,30 +15,43 @@ module PoolParty
     # TODO: Deprecate
     def with_cloud(cl, opts={}, &block)
       raise CloudNotFoundException.new("Cloud not found") unless cl
-      cl.options.merge!(opts) if opts
+      cl.dsl_options.merge!(opts) if opts
       cl.run_in_context &block if block
     end
     
     class Cloud < PoolParty::PoolPartyBaseClass
       attr_reader :templates, :cloud_name, :remote_base
+      attr_accessor :started_instance, :running_action
+      
+      # Default set of options. Most take the Default options from the default class
+      default_options(
+        {
+        :expand_when          => Default.expand_when,
+        :contract_when        => Default.contract_when,
+        :minimum_instances    => 2,
+        :maximum_instances    => 5,
+        :ec2_dir              => ENV["EC2_HOME"],
+        :minimum_runtime      => Default.minimum_runtime,
+        # :dependency_resolver  => ChefResolver,
+        :remoter_base         => Default.remoter_base,
+        :keypair              => nil,
+        :keypair_path         => nil,
+        :keypair_name         => nil
+        }.merge(Remote::Ec2.default_options)
+      )
 
       include CloudResourcer
-      include PoolParty::PluginModel
       include PoolParty::Resources
       include PoolParty::Callbacks
       include PoolParty::DependencyResolverCloudExtensions
       include PrettyPrinter
 
       # Net methods
-      include ::PoolParty::Remote
+      include PoolParty::Remote
       include PoolParty::CloudDsl
       include PoolParty::Verification
       # include PoolParty::Monitors
 
-      def verbose
-        true
-      end
-      
       def self.immutable_methods
         [:name]
       end
@@ -51,27 +63,28 @@ module PoolParty
       
       alias :name :cloud_name
       
-      # Call the remoter commands on the cloud if they don't exist on the cloud itself
-      # This gives the cloud access to the remote_base's methods
+      # Call the remoter commands on the remoter_base if they don't exist on the cloud itself.
+      # This gives the cloud access to the remote_base's methods.
       def method_missing(m, *args, &block)
-        remote_base.respond_to?(m) ? remote_base.send(m, *args, &block) : super
+        if remote_base.respond_to?(m)
+          remoter_opts = dsl_options.merge(remote_base.dsl_options).choose do |k,v|
+             remote_base.dsl_options.has_key?(k)
+          end
+          if args.size==1 && args.first.respond_to?(:merge)
+            new_args = [remoter_opts.merge(args.first)]
+          else
+            new_args = args.push(remoter_opts)
+          end
+          remote_base.send(m, *(new_args), &block)
+        else
+          super
+        end
       end
       
-      # Default set of options. Most take the Default options from the default class
-      default_options(
-        :expand_when => Default.expand_when,
-        :contract_when => Default.contract_when,
-        :minimum_instances => 2,
-        :maximum_instances => 5,
-        :access_key => Default.access_key,
-        :secret_access_key => Default.secret_access_key,
-        :ec2_dir => ENV["EC2_HOME"],
-        :minimum_runtime => Default.minimum_runtime,
-        :user => Default.user
-      )
-      
       additional_callbacks [
-        "after_launch_instance"
+        "after_launch_instance",
+        "before_provision",
+        "after_provision"
       ]
       
       # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
@@ -80,7 +93,9 @@ module PoolParty
         @cloud_name = name
         @cloud_name.freeze
         
-        plugin_directory "#{pool_specfile ? ::File.dirname(pool_specfile) : Dir.pwd}/plugins"        
+        setup_callbacks
+        
+        plugin_directory "#{pool_specfile ? ::File.dirname(pool_specfile) : Dir.pwd}/plugins"
         before_create
         super
         after_create
@@ -91,10 +106,8 @@ module PoolParty
         @cloud_name ||= @cloud_name ? @cloud_name : (args.empty? ? :default_cloud : args.first)
       end
       
-      def before_create     
-        context_stack.push self
-        (parent ? parent : self).add_poolparty_base_requirements
-        context_stack.pop
+      def before_create
+        add_poolparty_base_requirements
       end
       
       # Callback
@@ -102,7 +115,7 @@ module PoolParty
       # here the base requirements are added as well as an empty chef recipe is called
       # Also, the after_create hook on the plugins used by the cloud are called here
       def after_create
-        ::FileUtils.mkdir_p("#{Default.tmp_path}/dr_configure")
+        ::FileUtils.mkdir_p("#{tmp_path}/dr_configure")
         
         run_in_context do
           add_optional_enabled_services
@@ -112,30 +125,86 @@ module PoolParty
         
         plugin_store.each {|a| a.call_after_create_callbacks }
         setup_defaults
-        
-        setup_callbacks
+      end
+      
+      # CALLBACKS
+      # To hook into the chain of processes that PoolParty uses
+      # when launching, bootstrapping, configuring tasks, call
+      # these methods in your clouds.rb.
+      # Example:
+      # 
+      #   after :bootstrap do
+      #     stuff
+      #   end
+      # 
+      # These store the blocks into the after/before_blocks on self
+      # and are called as appropriate at the runtime
+      def after time, &block 
+        (after_blocks[time.to_sym] ||= []) << block
+      end      
+      def before time, &block
+        (before_blocks[time.to_sym] ||= []) << block
       end
       
       # setup defaults for the cloud
       def setup_defaults
-        # this can be overridden in the spec, but ec2 is the default
-        using :ec2
-        options[:keypair] ||= keypair rescue nil
-        options[:rules] = {:expand => dsl_options[:expand_when], :contract => dsl_options[:contract_when]}
-        dependency_resolver 'chef'        
-        # enable :haproxy unless dsl_options[:haproxy] == :disabled
+        set_vars_from_options(:keypair_name => key.basename, :keypair_path => key.full_filepath) rescue nil
+        
+        dsl_options[:rules] = {:expand   => "#{dsl_options[:expand_when]}", 
+                               :contract => dsl_options[:contract_when]}        
+        
+        dependency_resolver 'chef'
+        using Default.remoter_base unless @remote_base
       end
       
       def after_launch_instance(inst=nil)
         remote_base.send :after_launch_instance, inst
       end
       
+      # Keypairs
+      # Use the keypair path
+      def keypair(*args)
+        if args && !args.empty?
+          args.each do |arg|
+            unless arg.nil? || _keypair_filepaths.include?(arg)
+              k = arg.is_a?(Key) ? arg : Key.new(arg)
+              _keypairs.unshift k
+            end
+          end
+          self.keypair
+        else
+          unless @keypair
+            @keypair = _keypairs.select {|key| key.exists? }.first
+            self.keypair_path = @keypair.full_filepath
+            self.keypair_name = @keypair.basename
+            self.keypair = @keypair
+          end
+          @keypair
+        end
+      end
+
+      alias :set_keypairs :keypair
+      alias :key :keypair
+
+      def _keypairs
+        @keypairs ||= [Key.new]
+      end
+
+      # Collect the filepaths of the already loaded keypairs
+      def _keypair_filepaths
+        _keypairs.map {|a| a.filepath }
+      end
+
+      #TODO: deprecate: use key.full_filepath    
+      def full_keypair_path
+        @full_keypair_path ||= keypair.full_filepath
+      end
+      
       # provide list of public ips to get into the cloud
       def ips
         nodes(:status => "running").map {|ri| ri.ip }
       end
       
-      # TODO: make this be a random ip, since we should not rely on it being the same each time
       def ip
         ips.first
       end
@@ -171,15 +240,15 @@ module PoolParty
         end
       end
 
-      #FIXME MOVE TO DEPENDECY RESOL
+      #FIXME MOVE TO DEPENDENCY RESOLVER
       # Configuration files
       def build_manifest
         vputs "Building manifest"
         @build_manifest ||= build_from_existing_file
-        unless @build_manifest          
+        unless @build_manifest
           props = to_properties_hash
           
-          @build_manifest =  options[:dependency_resolver].send(:compile, props, self)
+          @build_manifest = dependency_resolver.send(:compile, props, self)
         end
         dputs "Finished creating manifest"
         @build_manifest
@@ -197,7 +266,7 @@ module PoolParty
         ::FileTest.file?("#{Default.base_config_directory}/poolparty.pp") ? open("#{Default.base_config_directory}/poolparty.pp").read : nil
       end
       
-      def write_properties_hash(filename=::File.join(Default.tmp_path, Default.properties_hash_filename) )
+      def write_properties_hash(filename=::File.join(tmp_path, Default.properties_hash_filename) )
         file_path = ::File.dirname(filename)
         file_name = "#{::File.basename(filename, ::File.extname(filename))}_#{name}#{::File.extname(filename)}"
         output = to_properties_hash.to_json
@@ -205,32 +274,126 @@ module PoolParty
         true
       end
       
-      # Add all the poolparty requirements here
-      # NOTE: These are written as plugins in the lib/poolparty/base_packages directory
-      # for examples. 
-      # Also note that there is no block associated. This is because we have written
-      # all that is necessary in a method called enable
-      # which is called when there is no block
-      def add_poolparty_base_requirements
-        # poolparty_base_heartbeat
-        poolparty_base_ruby
-        poolparty_base_packages        
+      def to_hash
+        hsh = to_properties_hash
+        hsh[:options].merge!({:remote_base => remote_base.to_hash})  
+        hsh
+      end
+      
+      def to_h
+        to_hash
       end
       
-      # TODO: Deprecate
-      def other_clouds
-        arr = []
-        clouds.each do |name, cl|
-          arr << cl if name != self.name
+      # TODO: test
+      # ruby -rrubygems -e 'require "poolparty";puts Cloud.load_from_json(open("/etc/poolparty/clouds.json").read).minimum_instances'      
+      def self.load_from_json(str, o={})
+        if ::File.file?(str)
+          str = open(str).read
+        end
+        parsed = JSON.parse(str).symbolize_keys!
+        opts = parsed.options
+        opts["keypair"] = opts["keypair_path"] = opts["keypair_name"]
+
+        cld = Cloud.new opts.cloud_name.to_sym
+        cld.dsl_options.merge! opts
+        cld.using opts.remoter_base.to_sym, opts.remote_base
+        cld.dependency_resolver opts[:dependency_resolver]
+        
+        #TODO: will never run
+        unless true || o[:full_stack]
+          context_stack.push cld
+          cld.ordered_resources = parsed.resources.map do |r|
+            case typ = r.delete(:pp_type)
+            when "plugin"
+              # This may become a problem on the server where the plugins
+              # cannot be found. TODO: Fix?!? How? Uh... fake plugin maybe?            
+              cld.send(r[:name].gsub(/_class/, '').to_sym) do              
+                set_vars_from_options(r)
+              end            
+            else
+              cld.send("has_#{typ}".to_sym, r)
+            end
+            context_stack.pop
+          end
+        else
+          cld.ordered_resources = parsed.resources
         end
-        arr
+        cld
       end
       
+      def remote_base(n=nil)
+        if n.nil?
+          @remote_base
+        else
+          @remote_base = n
+        end
+      end
+      
+      def tmp_path
+        Default.tmp_path / pool.name / name
+      end
+            
       # Reset the entire cloud
       def reset!
         reset_remoter_base!
         @build_manifest = @describe_instances = @remote_instances_list = nil
-      end            
+      end
+      
+      # Call before and after configure callbacks
+      # on the cloud.
+      # These are called from the dynamically defined callback method
+      #   call_before/after_configure/bootstrap_callbacks
+      # from within callbacks.rb
+      # If there is a callback block defined for the specific runtime
+      # method being called, then the appropriate callback blocks
+      # will be accessed and called from within the before/after_ callback
+      def before_bootstrap
+        before_blocks[:bootstrap].each {|b| b.call(self) } if before_blocks.has_key?(:bootstrap)
+      end
+      def after_bootstrap
+        after_blocks[:bootstrap].each {|b| b.call(self) } if after_blocks.has_key?(:bootstrap)
+      end
+      def before_provision
+        before_blocks[:provision].each {|b| b.call(self) } if before_blocks.has_key?(:provision)
+      end
+      def after_provision
+        after_blocks[:provision].each {|b| b.call(self) } if after_blocks.has_key?(:provision)
+      end
+      def before_configure
+        before_blocks[:configure].each {|b| b.call(self) } if before_blocks.has_key?(:configure)
+      end
+      def after_configure
+        after_blocks[:configure].each {|b| b.call(self) } if after_blocks.has_key?(:configure)
+      end
+      
+      private
+      
+      # Storage for the after and before callback blocks
+      # when calling the callbacks in the cloud. They are stored as hashes with
+      # and are filled with arrays of blocks during load-time
+      def after_blocks
+        @after_blocks ||= {}
+      end
+      def before_blocks
+        @before_blocks ||= {}
+      end
+      
+      def pool
+        parent && parent.is_a?(Pool) ? parent : self
+      end
+      
+      # Add all the poolparty requirements here
+      # NOTE: These are written as plugins in the lib/poolparty/plugins directory
+      # for examples. 
+      # Also note that there is no block associated. This is because we have written
+      # all that is necessary in a method called enable
+      # which is called when there is no block
+      def add_poolparty_base_requirements
+        # poolparty_base_heartbeat
+        ruby
+        pool_party_base_packages
+      end
+                
     end
   end 
 end

data/lib/poolparty/poolparty/default.rb

@@ -3,18 +3,19 @@
   This handles user interaction, loading the parameters, etc.
 =end
 require "open-uri"
-require "ftools"
+require "fileutils"
 module PoolParty
   class Default
     include Dslify
     
     # def self.options
     default_options(
+      :verbose => false,
       :testing => false,
-      :debugging => false,
       :minimum_instances => 2,
       :maximum_instances => 5,
       :user => "root", # This should change here
+      :keypair_name => nil,
       :base_keypair_path => "#{ENV["HOME"]}/.ec2",
       :base_ssh_path => "#{ENV["HOME"]}/.ssh",
       :tmp_path => "/tmp/poolparty",
@@ -29,6 +30,7 @@ module PoolParty
       :default_specfile_name => "clouds.rb",
       :properties_hash_filename => "clouds.json",
       :vendor_path => "#{::File.dirname(__FILE__)}/../../../vendor",
+      :poolparty_src_path => "#{::File.dirname(__FILE__)}/../../..",
       :port => "80",
       :forwarding_port => "8080",
       :monitor_port => 8081,
@@ -36,22 +38,22 @@ module PoolParty
       :butterfly_port => 8642,
       :minimum_runtime  => 3000, #50.minutes in seconds
       :contract_when => "load < 0.25",
-      :expand_when => "load > 0.9"
+      :expand_when => "load > 0.9",
+      :ec2_dir => ENV["EC2_HOME"], #TODO: move to ec2 class
+      :image_id => nil,
+      :access_key => nil,
+      :secret_access_key => nil,
+      :remoter_base => :ec2,
+      :availabilty_zone => 'us-east-1a'
     )
     
-    def options
-      default_options
-    end
     
     # Class methods
     class << self
       def method_missing(m,*a,&block)
-        default_options.include?(m) ? default_options[m] : super
-      end
-      def options
-        default_options
+        dsl_options.include?(m) ? dsl_options[m] : super
       end
-      # Get the access_key
+      # # Get the access_key
       def access_key
         @access_key ||= load_access_keys_from_environment_var || load_keys_from_file[:access_key]
       end
@@ -72,11 +74,14 @@ module PoolParty
       end
       # Store the keys in a yaml format to give the master access
       # So that the master has access to the files
-      def store_keys_in_file
+      def store_keys_in_file(f=nil)
         unless access_key.nil? || secret_access_key.nil?
-          write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
+          write_to_file( (f ? f : key_file_locations.first), keys_in_yaml)
         end
       end
+      def keys_in_yaml
+        YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key})
+      end
       def store_keys_in_file_for(obj=nil)
         if obj
           @access_key = obj.access_key
@@ -98,7 +103,9 @@ module PoolParty
         [
           ".ppkeys",
           "#{Default.base_config_directory}/.ppkeys",
-          "#{Default.storage_directory}/ppkeys",          
+          "#{Default.storage_directory}/ppkeys",
+          "#{ENV["HOME"]}/.ssh/ppkeys",
+          "#{ENV["HOME"]}/.ssh/.ppkeys",
           "~/.ppkeys",
           "ppkeys"
         ]
@@ -117,7 +124,7 @@ module PoolParty
             "/var/poolparty"
         ].select do |dir|
           dir if viable_directory?(dir)
-        end.first || ::File.join( "/tmp/poolparty")
+        end.first || ::File.join( "/tmp/poolparty/#{name}")
       end
       def logger_location
         [

data/lib/poolparty/poolparty/key.rb

@@ -3,6 +3,8 @@
 =end
 module PoolParty
   class Key
+    include SearchablePaths
+    has_searchable_paths(:dirs => ["/", "keys"], :prepend_paths => ["#{ENV["HOME"]}/.ssh"])
     
     attr_accessor :filepath
     
@@ -24,10 +26,25 @@ module PoolParty
     # Returns the full_filepath of the key. If a full filepath is passed, we just return the expanded filepath
     # for the keypair, otherwise query where it is against known locations
     def full_filepath
-      @full_filepath ||= ::File.file?(::File.expand_path(filepath)) ? ::File.expand_path(filepath) : search_in_known_locations
+      @full_filepath ||= ::File.file?(::File.expand_path(filepath)) ? ::File.expand_path(filepath) : search_in_known_locations(filepath)
     end
     alias :to_s :full_filepath
     
+    #TODO: gracefully handle the case when a passpharase is needed
+    # Generate a public key from the private key
+    def public_key
+      if !@public_key_string || @public_key_string.empty?
+         @public_key_string = `ssh-keygen -y -f #{full_filepath}`
+         raise 'Unable to generate public_key_string' if @public_key_string.empty?
+      else
+        @public_key_string
+      end
+    end
+    
+    def public_key=(str)
+       @public_key_string = str
+    end
+    
     # Basename of the keypair
     def basename
       @basename ||= ::File.basename(full_filepath, ::File.extname(full_filepath)) rescue filepath
@@ -38,39 +55,13 @@ module PoolParty
       @filename ||= ::File.basename(full_filepath) rescue filepath
     end
     
-    # Search for the key in default locations with the entire filepath
-    # if the file exists. If it doesn't exist in the default locations, 
-    # then it returns nil and assumes we it doesn't exist
-    def search_in_known_locations
-      self.class.keypair_paths.each do |path|
-        full_path = ::File.join( ::File.expand_path(path), ::File.basename(filepath))
-        return full_path if ::File.exists?(full_path)
-      end
-      # raise Exception.new("We cannot continue without a keypair. Please define a keypair in your clouds.rb")
-      # TODO: Add raise for keypair
-      nil
-    end
-    
-    # Default locations to search for the key
-    def self.keypair_paths
-      [ 
-        "#{ENV["HOME"]}/.ssh",
-        "#{Default.poolparty_home_path}/keys",
-        PoolParty::Default.base_keypair_path,
-        PoolParty::Default.base_config_directory,
-        PoolParty::Default.base_ssh_path,
-        PoolParty::Default.remote_storage_path,
-        Dir.pwd
-      ]
-    end
-    
     # Support to add the enumerable each to keys
     def each
       yield full_filepath
     end
     
     # Turn the keypair into the a useful json string
-    def to_json
+    def to_hash
       "{\"basename\":\"#{basename}\",\"full_filepath\": \"/etc/poolparty/#{filename}\"}"
     end
     

data/lib/poolparty/poolparty/neighborhoods.rb

@@ -10,7 +10,7 @@ require "#{::File.dirname(__FILE__)}/../schema"
 
 module PoolParty
   class Neighborhoods
-    include ::PoolParty::Pinger    
+    include ::PoolParty::Pinger
     attr_reader :schema
     
     # Create a neighborhood from a string, array or hash given.
@@ -20,7 +20,7 @@ module PoolParty
       when Array
         {:instances => data.map {|entry| disect(entry) }}
       when String
-        {:instances => JSON.parse(data)}#.map "#{inst["instance_id"]}\t#{inst["ip"]}"}}
+        JSON.parse(data)#.map "#{inst["instance_id"]}\t#{inst["ip"]}"}}
       when Hash
         data
       end
@@ -30,7 +30,7 @@ module PoolParty
     
     # Get the known instances from the neighborhood.json file on the server
     def instances
-      @instances ||= @schema.to_hash[:instances] rescue @schema.instances.collect {|line| disect(line) }
+      @instances ||= @schema.to_hash[:instances] #rescue @schema.instances.collect {|line| disect(line) }
     end
     
     # Returns empty if the neighborhood has no instances
@@ -92,9 +92,20 @@ module PoolParty
         new( open("/etc/poolparty/neighborhood.json").read )
       elsif ping_port("127.0.0.1", Default.butterfly_port, 1)# butterfly responding?
         require "open-uri"
-        new( open("http://127.0.0.1:8642/neighborhood").read )
+        begin
+          timeout(2) do
+            new( open("http://127.0.0.1:8642/neighborhood").read )
+          end
+        rescue TimeoutError => e
+          require "#{::File.dirname(__FILE__)}/../../poolparty"
+          cld = ::PoolParty::Cloud::Cloud.load_from_json(open("/etc/poolparty/clouds.json").read)          
+          nodes = cld.nodes({:status => "running"}, false)
+          data = nodes.map {|hsh| hsh.reject {|k,v| v.nil? }}.map {|a| a.merge(:launching_time => a[:launching_time].to_s) }
+          # ::File.open("/etc/poolparty/neighborhood.json", "w") {|f| f << "{\"instances\":#{data.to_json}}" }
+          new(data)
+        end
       else
-        new("[]")
+        new("{\"instances\":[]}")
       end
     end