auser-poolparty 1.2.4 → 1.2.7

data/lib/poolparty/net/remoter/connections.rb

@@ -1,4 +1,3 @@
-require "#{::File.dirname(__FILE__)}/../../modules/pinger"
 require 'rubygems'
 require 'net/ssh'
 

data/lib/poolparty/net/remoter/interactive.rb

@@ -2,18 +2,18 @@ module PoolParty
   module Remote
     
     # Select a list of instances based on their status
-    def nodes(hsh={})
-      list_of_instances.select_with_hash(hsh)
+    def nodes(hsh={}, with_neighborhood_default=true)
+      list_of_instances(with_neighborhood_default).select_with_hash(hsh)
     end
     
     # Select the list of instances, either based on the neighborhoods
     # loaded from /etc/poolparty/neighborhood.json
     # or by the remote_base on keypair
-    def list_of_instances
+    def list_of_instances(with_neighborhood_default=true)
       return @list_of_instances if @list_of_instances
       @containing_cloud = self
-      n = Neighborhoods.load_default
-      @list_of_instances = (n.instances.empty? ? _list_of_instances : n.instances)
+      n = with_neighborhood_default ? Neighborhoods.load_default : nil
+      @list_of_instances = ((n.nil? || n.instances.empty?) ? _list_of_instances : n.instances)
     end
 
     private
@@ -24,7 +24,7 @@ module PoolParty
     
     # List the instances for the current key pair, regardless of their states
     # If no keypair is passed, select them all
-    def _list_of_instances(select={})
+    def _list_of_instances(select={})      
       @describe_instances ||= remote_base.describe_instances(dsl_options).select_with_hash(select)
     end
     

data/lib/poolparty/net/remoter_base.rb

@@ -16,7 +16,6 @@
     register_remote_base :remote_base_name
   
 =end
-require "#{::File.dirname(__FILE__)}/../modules/pinger.rb"  #FIXME should not need this here
 module PoolParty
 
   module Remote    
@@ -29,7 +28,8 @@ module PoolParty
       
       dsl_methods :cloud,                # The cloud this remoter_base is a part of
                   :keypair,
-                  :image_id
+                  :image_id,
+                  :keypair_name
         
       def initialize(opts={}, &block)
         opts.each {|k,v| opts[k] = v.call if v.respond_to?(:call) }
@@ -187,6 +187,14 @@ module PoolParty
       def before_shutdown
       end
       
+      def to_s
+        self.class.name
+      end
+      
+      def to_hash
+        dsl_options
+      end
+      
     end
     
   end

data/lib/poolparty/net/remoter_bases/ec2/ec2.rb

@@ -47,8 +47,13 @@ module PoolParty
         :availability_zone => "us-east-1a",
         :access_key => ENV['AWS_ACCESS_KEY'],
         :secret_access_key => ENV['AWS_SECRET_ACCESS_KEY'],
-        :security_group => ["default"]
+        :security_group => ["default"],
+        :keypair_name =>nil
         })
+        
+      def ami
+        image_id
+      end
       
       # Requires a hash of options
       def self.launch_new_instance!(o)
@@ -58,8 +63,9 @@ module PoolParty
       # TODO: Fix the key_name issue
       # Start a new instance with the given options
       def launch_new_instance!(o={})
+        set_vars_from_options o
         raise "You must pass a keypair to launch an instance, or else you will not be able to login. options = #{o.inspect}" if !keypair
-        o.merge!( dsl_options ).merge!(:key_name=>keypair.basename)
+        o.merge!( dsl_options.merge(:key_name=>keypair_name) )
         instance = ec2(o).run_instances(o)
         begin
           h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
@@ -83,7 +89,7 @@ module PoolParty
       def describe_instances(o={})
         id = 0
         set_vars_from_options(dsl_options.merge(o))
-        get_instances_description(dsl_options).each_with_index do |h,i|          
+        get_instances_description(dsl_options).each_with_index do |h,i|
           if h[:status] == "running"
             inst_name = id == 0 ? "master" : "node#{id}"
             id += 1
@@ -106,8 +112,8 @@ module PoolParty
       
       # return or create a new base EC2 connection object that will actually connect to ec2
       def ec2(o={})
-        @ec2 ||= EC2::Base.new( :access_key_id => get_access_key, 
-                                :secret_access_key => get_secret_access_key
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key] || get_access_key, 
+                                :secret_access_key => o[:secret_access_key] || get_secret_access_key
                               )
       end
       def self.ec2(o)
@@ -119,8 +125,10 @@ module PoolParty
       # Get the ec2 description for the response in a hash format
       def get_instances_description(o={})
         #TODO: only use keypair.full_filepath
+        set_vars_from_options dsl_options.merge(o)
         key_hash = {:keypair => self.keypair_name}
-        EC2ResponseObject.get_descriptions(ec2(dsl_options).describe_instances).select_with_hash(key_hash)
+        out = EC2ResponseObject.get_descriptions(ec2(dsl_options).describe_instances)
+        out = keypair_name ? out.select_with_hash(key_hash) : out
       end
       def get_descriptions(o={})
         self.class.get_descriptions(o)

data/lib/poolparty/net/remoter_bases/metavirt/metavirt.rb

@@ -8,7 +8,7 @@ module PoolParty
   module Remote
     class Metavirt < Remote::RemoterBase
       include Dslify
-      include CloudResourcer
+      include ::PoolParty::CloudResourcer
       
       default_options(
         # :machine_image => 'ubuntu-kvm',

data/lib/poolparty/plugins/apache2/apache.rb

@@ -26,6 +26,10 @@ default host.
       def before_load(o={}, &block)
         install
       end
+      
+      def www_user(www_user_name='www-data')
+        www_user_name
+      end
             
       def install
         installed_as_worker
@@ -224,20 +228,21 @@ default host.
     def virtual_host_entry(file)
       @virtual_host_entry = true
       if ::File.file?(file)
-        has_file(options.merge({:name => "/etc/apache2/sites-available/#{name}", 
+        has_file(dsl_options.merge({:name => "/etc/apache2/sites-available/#{name}", 
                                 :template => file, 
                                 :requires => get_package("apache2")}))
       else
-        has_file(options.merge({:content => file, 
+        has_file(dsl_options.merge({:content => file, 
                                 :name => "/etc/apache2/sites-available/#{name}", 
                                 :requires => get_package("apache2")}))
       end
     end
+
     
     def loaded(opts={}, parent=self)
-      has_directory(:name => "/var/www")
-      has_directory(:name => "/var/www/#{name}")
-      has_directory(:name => "/var/www/#{name}/logs", :owner => "www-data")
+      has_directory(:name => "/var/www", :owner => www_user, :mode=>'0744')
+      has_directory(:name => "/var/www/#{name}", :owner => www_user, :mode=>'0744')
+      has_directory(:name => "/var/www/#{name}/logs", :owner => www_user, :mode=>'0744')
 
       has_variable(:name => "sitename", :value => "#{name}")
 
@@ -264,8 +269,11 @@ eof
   virtual_resource(:passengersite) do    # {{{
 
     default_options(
-      :dir => "/var/www",
-      :appended_path => nil
+      :dir            => "/var/www",
+      :appended_path  => nil,
+      :owner          => 'www-data', 
+      :mode           =>'0744',
+      :enviornment    => 'production'
     )
 
     def loaded(opts={}, prnt=nil)
@@ -278,7 +286,7 @@ eof
 <VirtualHost *:#{port}>
     ServerName #{name}
     DocumentRoot #{site_directory}/public
-    RailsEnv production
+    RailsEnv #{enviornment}
     ErrorLog #{site_directory}/log/error_log
     CustomLog #{site_directory}/log/access_log common
 </VirtualHost>

data/lib/poolparty/poolparty/cloud.rb

@@ -16,7 +16,7 @@ module PoolParty
     # TODO: Deprecate
     def with_cloud(cl, opts={}, &block)
       raise CloudNotFoundException.new("Cloud not found") unless cl
-      cl.options.merge!(opts) if opts
+      cl.dsl_options.merge!(opts) if opts
       cl.run_in_context &block if block
     end
     
@@ -130,9 +130,9 @@ module PoolParty
       
       # setup defaults for the cloud
       def setup_defaults
-        set_vars_from_options(:keypair_name => key.basename, 
-                              :keypair_path => key.full_filepath)        
-        dsl_options[:rules] = {:expand   => dsl_options[:expand_when], 
+        set_vars_from_options(:keypair_name => key.basename, :keypair_path => key.full_filepath) rescue nil
+        
+        dsl_options[:rules] = {:expand   => "#{dsl_options[:expand_when]}", 
                                :contract => dsl_options[:contract_when]}        
         
         set_dependency_resolver 'chef'
@@ -257,7 +257,24 @@ module PoolParty
       end
       
       def to_json
-        to_properties_hash.to_json
+        to_properties_hash.reject{|k,v| k == :remote_base }.to_json
+      end
+      
+      # TODO: test
+      # ruby -rrubygems -e 'require "poolparty";puts Cloud.load_from_json(open("/etc/poolparty/clouds.json").read).minimum_instances'      
+      def self.load_from_json(str)
+        parsed = JSON.parse(str).each {|k,v| dsl_options[k.to_sym] = v}
+        opts= parsed.options
+        opts["keypair"] = opts["keypair_path"] = opts["keypair_name"]
+        # cld.remoter_base = PoolParty::Remote.module_eval( schema.options.remoter_base.camelcase )
+        # opts.remoter_base_class = PoolParty::Remote.module_eval( opts.remoter_base.camelcase )
+        # opts.remoter_base_class.new opts.remote_base
+        opts["dependency_resolver"] = options.dependency_resolver.send(:new, opts)
+        cld = Cloud.new opts.cloud_name.to_sym
+        cld.dsl_options.merge opts
+        cld.using opts.remoter_base.to_sym
+        cld.dsl_options.symbolize_keys!
+        cld
       end
       
       def tmp_path

data/lib/poolparty/poolparty/default.rb

@@ -17,6 +17,7 @@ module PoolParty
       :minimum_instances => 2,
       :maximum_instances => 5,
       :user => "root", # This should change here
+      :keypair_name => nil,
       :base_keypair_path => "#{ENV["HOME"]}/.ec2",
       :base_ssh_path => "#{ENV["HOME"]}/.ssh",
       :tmp_path => "/tmp/poolparty",
@@ -55,18 +56,18 @@ module PoolParty
         dsl_options.include?(m) ? dsl_options[m] : super
       end
       # # Get the access_key
-      # def access_key
-      #   @access_key ||= load_access_keys_from_environment_var || load_keys_from_file[:access_key]
-      # end
-      # def load_access_keys_from_environment_var
-      #   [ ENV["AWS_ACCESS_KEY"], ENV["AWS_ACCESS_KEY_ID"]].reject {|a| a.nil? }.first
-      # end
-      # def secret_access_key
-      #   @secret_access_key ||= load_secret_access_keys_from_environment_var || load_keys_from_file[:secret_access_key]
-      # end
-      # def load_secret_access_keys_from_environment_var
-      #   [ ENV["AWS_SECRET_ACCESS_KEY"] ].reject {|a| a.nil? }.first
-      # end
+      def access_key
+        @access_key ||= load_access_keys_from_environment_var || load_keys_from_file[:access_key]
+      end
+      def load_access_keys_from_environment_var
+        [ ENV["AWS_ACCESS_KEY"], ENV["AWS_ACCESS_KEY_ID"]].reject {|a| a.nil? }.first
+      end
+      def secret_access_key
+        @secret_access_key ||= load_secret_access_keys_from_environment_var || load_keys_from_file[:secret_access_key]
+      end
+      def load_secret_access_keys_from_environment_var
+        [ ENV["AWS_SECRET_ACCESS_KEY"] ].reject {|a| a.nil? }.first
+      end
       def read_keyfile
         open(get_working_key_file_locations).read
       end
@@ -75,11 +76,14 @@ module PoolParty
       end
       # Store the keys in a yaml format to give the master access
       # So that the master has access to the files
-      def store_keys_in_file
+      def store_keys_in_file(f=nil)
         unless access_key.nil? || secret_access_key.nil?
-          write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
+          write_to_file( (f ? f : key_file_locations.first), keys_in_yaml)
         end
       end
+      def keys_in_yaml
+        YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key})
+      end
       def store_keys_in_file_for(obj=nil)
         if obj
           @access_key = obj.access_key
@@ -101,7 +105,9 @@ module PoolParty
         [
           ".ppkeys",
           "#{Default.base_config_directory}/.ppkeys",
-          "#{Default.storage_directory}/ppkeys",          
+          "#{Default.storage_directory}/ppkeys",
+          "#{ENV["HOME"]}/.ssh/ppkeys",
+          "#{ENV["HOME"]}/.ssh/.ppkeys",
           "~/.ppkeys",
           "ppkeys"
         ]

data/lib/poolparty/poolparty/key.rb

@@ -61,7 +61,7 @@ module PoolParty
     end
     
     # Turn the keypair into the a useful json string
-    def to_json
+    def to_hash
       "{\"basename\":\"#{basename}\",\"full_filepath\": \"/etc/poolparty/#{filename}\"}"
     end
     

data/lib/poolparty/poolparty/neighborhoods.rb

@@ -20,7 +20,7 @@ module PoolParty
       when Array
         {:instances => data.map {|entry| disect(entry) }}
       when String
-        {:instances => JSON.parse(data)}#.map "#{inst["instance_id"]}\t#{inst["ip"]}"}}
+        JSON.parse(data)#.map "#{inst["instance_id"]}\t#{inst["ip"]}"}}
       when Hash
         data
       end
@@ -30,7 +30,7 @@ module PoolParty
     
     # Get the known instances from the neighborhood.json file on the server
     def instances
-      @instances ||= @schema.to_hash[:instances] rescue @schema.instances.collect {|line| disect(line) }
+      @instances ||= @schema.to_hash[:instances] #rescue @schema.instances.collect {|line| disect(line) }
     end
     
     # Returns empty if the neighborhood has no instances
@@ -92,9 +92,20 @@ module PoolParty
         new( open("/etc/poolparty/neighborhood.json").read )
       elsif ping_port("127.0.0.1", Default.butterfly_port, 1)# butterfly responding?
         require "open-uri"
-        new( open("http://127.0.0.1:8642/neighborhood").read )
+        begin
+          timeout(2) do
+            new( open("http://127.0.0.1:8642/neighborhood").read )
+          end
+        rescue TimeoutError => e
+          require "#{::File.dirname(__FILE__)}/../../poolparty"
+          cld = ::PoolParty::Cloud::Cloud.load_from_json(open("/etc/poolparty/clouds.json").read)          
+          nodes = cld.nodes({:status => "running"}, false)
+          data = nodes.map {|hsh| hsh.reject {|k,v| v.nil? }}.map {|a| a.merge(:launching_time => a[:launching_time].to_s) }
+          # ::File.open("/etc/poolparty/neighborhood.json", "w") {|f| f << "{\"instances\":#{data.to_json}}" }
+          new(data)
+        end
       else
-        new("[]")
+        new("{\"instances\":[]}")
       end
     end
     

data/lib/poolparty/poolparty/pool.rb

@@ -11,7 +11,7 @@ module PoolParty
     
     def with_pool(pl, opts={}, &block)
       raise CloudNotFoundException.new("Pool not found") unless pl
-      pl.options.merge!(opts) if pl.options
+      pl.dsl_options.merge!(opts) if pl.dsl_options
       pl.run_in_context &block if block
     end
     

data/lib/poolparty/poolparty/resource.rb

@@ -15,6 +15,7 @@ module PoolParty
     end
     
     class Resource < PoolParty::PoolPartyBaseClass
+      include JSON
       attr_accessor :prestring, :poststring
       
       # include CloudResourcer

data/lib/poolparty/provision/boot_strapper.rb

@@ -28,6 +28,7 @@ module PoolParty
                           grempe-amazon-ec2
                           ohai
                           chef
+                          ruby-openid
                           adamwiggins-rest-client
                           rack
                           thin
@@ -83,6 +84,8 @@ module PoolParty
       def pack_the_dependencies
         # Add the keypair to the instance... shudder
         ::Suitcase::Zipper.add(keypair, "keys")
+        ::Suitcase::Zipper.add_content_as(Default.keys_in_yaml, "ppkeys", "keys")
+        
         edge_pp_gem = Dir["#{Default.vendor_path}/../pkg/*poolparty*gem"].pop
         # Use the locally built poolparty gem if it is availabl
             if edge_pp_gem
@@ -108,9 +111,10 @@ module PoolParty
         ::Suitcase::Zipper.add("#{::File.join(File.dirname(__FILE__), '..', 'templates', 'gemrc_template' )}", "etc/poolparty")
         
         instances = @cloud.nodes(:status => "running") + [@cloud.started_instance]
-        ::Suitcase::Zipper.add_content_as(
-          {:instances => instances.flatten.compact}.to_json, 
-          "neighborhood.json", "/etc/poolparty")
+        # ::Suitcase::Zipper.add_content_as(
+        #   instances.flatten.compact.to_json, 
+        #   "neighborhood.json", "/etc/poolparty"
+        # )
         
         ::Suitcase::Zipper.build_dir!("#{cloud.tmp_path}/dependencies")
         
@@ -140,6 +144,7 @@ module PoolParty
           "ln -sfv /usr/bin/gem1.8 /usr/bin/gem", #TODO: check if this is really needed
           "cd ../ && rm -rf rubygems-1.3.1*",
           "gem source --add http://gems.github.com",
+          "gem sources -a http://gems.opscode.com",
           "cd /var/poolparty/dependencies/gems/",
           "gem install --no-rdoc --no-ri *.gem",
           "cd /var/poolparty/dependencies",
@@ -150,8 +155,9 @@ module PoolParty
           "chmod 600 /root/.ssh/#{keypair_name}",
           # "god -c /etc/poolparty/monitor.god",
           "mkdir -p /var/log/poolparty/",
+          "echo '-- Starting monitor_rack --'",
           "thin -R /etc/poolparty/monitor.ru -p 8642 --pid /var/run/stats_monitor.pid --daemon -l /var/log/poolparty/monitor.log start 2>/dev/null",
-          "tail /var/log/poolparty/monitor.log",
+          "tail -n 20 /var/log/poolparty/monitor.log",
           'echo "bootstrap" >> /var/poolparty/POOLPARTY.PROGRESS']
         commands << self.class.class_commands unless self.class.class_commands.empty?
       end

data/lib/poolparty/provision/dr_configure.rb

@@ -22,13 +22,13 @@ module PoolParty
      # In case the method is being called on ourself, let's check the 
      # defaults hash to see if it's available there
      def method_missing(m,*a,&block)
-       if self.class.defaults.has_key?(m) 
-         self.class.defaults[m]
-       elsif @cloud
+       # if self.class.defaults.has_key?(m) 
+       #   self.class.defaults[m]
+       # elsif @cloud
          @cloud.send m, *a, &block
-       else
-         super
-       end
+       # else
+         # super
+       # end
      end
     
      attr_reader :cloud, :keypair, :run_count, :cloud_name
@@ -36,7 +36,7 @@ module PoolParty
      def initialize(host, opts={}, &block)
        self.class.defaults.merge(opts).to_instance_variables(self)
        @target_host = host
-       @configurator = "::PoolParty::Provision::#{dependency_resolver.capitalize}".constantize
+       @configurator = "::PoolParty::Provision::#{dependency_resolver.camelcase}".constantize
        @cloud = opts[:cloud]
        @cloud_name = @cloud.name
        @keypair = @cloud.keypair
@@ -54,7 +54,7 @@ module PoolParty
        ::FileUtils.mkdir_p "#{cloud.tmp_path}/dr_configure" unless ::File.directory?("#{cloud.tmp_path}/dr_configure")
       ::File.cp $pool_specfile, "#{cloud.tmp_path}/dr_configure/clouds.rb"
       ::File.open "#{cloud.tmp_path}/dr_configure/clouds.json", "w" do |f|
-        f << cloud.to_properties_hash.to_json
+        f << cloud.to_json
       end
       
       setup_configurator
@@ -99,7 +99,7 @@ module PoolParty
          'chmod 644 /var/poolparty/dr_configure/clouds.rb',
          'cp /var/poolparty/dr_configure/clouds.json /etc/poolparty',
          'cp /var/poolparty/dr_configure/clouds.rb /etc/poolparty',
-         "touch /var/poolparty/POOLPARTY.PROGRESS",
+         'server-manage-election', #ensures that the monitor gets some data
          'echo "configure" >> /var/poolparty/POOLPARTY.PROGRESS'
          ]
        commands << self.class.class_commands unless self.class.class_commands.empty?