poolparty 0.2.69 → 0.2.84

data/lib/poolparty/core/string.rb

@@ -22,7 +22,7 @@ class String
     self.strip.split(/\n/).join(" && ")
   end
   def top_level_class
-    self.split("::")[-1].underscore.downcase rescue self
+    self.split("::")[-1].underscore.downcase rescue self.class.to_s
   end
   def sanitize
     self.gsub(/[ \.\/\-]*/, '')

data/lib/poolparty/dependency_resolutions/puppet.rb

@@ -1,4 +1,12 @@
 module PoolParty
+  module Resources
+    class Resource
+      def self.resource_string_name(n, key)
+        "#{n.to_s.sanitize.capitalize}['#{key}']"
+      end
+    end
+  end
+  
   module DependencyResolutions
     module Puppet
       
@@ -39,6 +47,7 @@ module PoolParty
       
       def resources_string_from_resources(res, pre="\t")
         @variables = res.extract! {|name,resource| name == :variable}
+        
         returning Array.new do |str|
           unless @variables.empty?
             str << "\n# Variables"
@@ -54,9 +63,8 @@ module PoolParty
       end
       
       def to_s
-        "#{class_type_name.capitalize}['#{key}']"
+        self.class.resource_string_name(class_type_name.capitalize, key)
       end
-      
     end
   end
 end

data/lib/poolparty/exceptions/ProvisionerException.rb

@@ -0,0 +1,5 @@
+class ProvisionerException < Exception  
+  def initialize(msg="Provisioner process failed")
+    @message = "Error: #{msg}"
+  end
+end

data/lib/poolparty/extra/deployments.rb

@@ -0,0 +1,31 @@
+module PoolParty
+  module Extra
+    class Deployments
+
+      class << self
+        
+        def include_deployment(filename)
+          return nil unless ::File.file? filename
+          name = ::File.basename(filename, ::File.extname(filename))
+          contents = open(filename).read
+
+          plugin_klass = PoolParty::PluginModel::PluginModel.new(name)
+          plugin_klass.klass.class_eval <<-EOE
+            def enable
+              #{contents}
+            end
+          EOE
+          plugin_klass
+        end
+        
+        def include_deployments(dir)
+          return nil unless ::File.directory? dir
+          Dir["#{dir}/*"].each do |fi|
+            include_deployment fi
+          end
+          dir
+        end
+      end
+    end    
+  end
+end

data/lib/poolparty/helpers/binary.rb

@@ -5,7 +5,7 @@ module PoolParty
   def load_pool(filename)
 
     unless filename && ::File.readable?(filename)
-      puts "Please specify your cloud with -s, move it to ./pool.spec or in your POOL_SPEC environment variable"
+      puts "Please specify your cloud with -s, move it to ./clouds.pool or in your POOL_SPEC environment variable"
       exit(1)
     else
       $pool_specfile = filename
@@ -13,6 +13,15 @@ module PoolParty
     end
   end
   
+  def print_with_nice_printer(header=nil, strs=[])
+    returning NicePrinter.new do |printer|
+      printer.header
+      printer.center(header) if header
+      strs.each {|st| printer << st if st}
+      printer.footer
+    end.print
+  end
+  
   def pool_specfile
     $pool_specfile
   end
@@ -38,6 +47,7 @@ module PoolParty
             "#{Base.default_specfile_name}",
             ENV["POOL_SPEC"],            
             "#{Base.storage_directory}/#{Base.default_specfile_name}",
+            "#{Base.base_config_directory}/#{Base.default_specfile_name}",
             "#{Base.default_project_specfile_name}"
         ].reject {|a| a.nil?}.reject do |f|
           f unless ::File.readable?(f)
@@ -47,7 +57,7 @@ module PoolParty
       def daemonize(&block)
         vputs "Daemonizing..."
         trap("CHLD") {Process.wait(-1, Process::WNOHANG)}        
-        fork do
+        pid = fork do
           Signal.trap('HUP', 'IGNORE') # Don't die upon logout
           File.open("/dev/null", "r+") do |devnull|
             $stdout.reopen(devnull)
@@ -56,6 +66,8 @@ module PoolParty
           end
           block.call if block
         end
+        Process.detach(pid)
+        pid
       end
       
     end

data/lib/poolparty/helpers/optioner.rb

@@ -9,21 +9,58 @@ module PoolParty
   class Optioner
     include Configurable
     include MethodMissingSugar
-        
-    def initialize(args=[], opts={}, &block)
+    
+    def initialize(args=[], opts={}, &block)      
+      boolean_args << opts[:boolean_args] if opts.has_key?(:boolean_args)
+
       @arguments = parse_args(args)
-      @parse_options = opts[:parse_options] ? opts[:parse_options] : true
       @extra_help = opts.has_key?(:extra_help) ? opts[:extra_help] : ""
-      @abstract = opts[:abstract] ? opts[:abstract] : false
-      @command = opts[:command] ? opts[:command] : false
+      @abstract = opts.has_key?(:abstract) ? opts[:abstract] : false
+      @load_pools = opts.has_key?(:load_pools) ? opts[:load_pools] : !@abstract
+      @parse_options = opts.has_key?(:parse_options) ? opts[:parse_options] : true
+      @command = opts.has_key?(:command) ? opts[:command] : false
       
       parse_options(&block) if @parse_options
       set_default_options
       self
     end
+    def daemonizeable
+      @opts.on('-d', '--daemonize', 'Daemonize starting the cloud')    { self.daemon true }
+    end
+    def cloudnames
+      @opts.on('-n cloudname', '--name name', 'Start cloud by this name')    { |c| self.cloudname c }
+    end 
+    def unflagged_args
+      @unflagged_args ||= []
+    end
+    def flagged_args
+      @flagged_args ||= []
+    end
+    def boolean_args
+      @boolean_args ||= ['-V', '-h', '-t', '-v', '--debug']
+    end
     
-    def parse_args(argv, safe=[])
-      argv
+    # Break ARGV into 2 arrays, one for flagged options one for unflagged
+    # For example the "command -v -i 1 five six -x"
+    # becomes ['-v', '-i', 1, '-x'] and ['five', 'six']
+    # Boolean options, such as -v, must be specified in the optioner definition
+    def parse_args(args=[])
+      i=0
+      while i < args.length
+        if boolean_args.include?(args[i])
+          flagged_args << args[i]
+        else
+          if args[i][0].chr == "-"
+            flagged_args << args[i]
+            flagged_args << args[i+1] if (args[i+1] and !args[i+1].nil?)
+            i+=1
+          else
+            unflagged_args << args[i]
+          end
+        end
+        i+=1
+      end
+      args
     end
     
     def parent
@@ -37,40 +74,42 @@ module PoolParty
     
     def parse_options(&blk)
       progname = $0.include?("-") ? "#{::File.basename($0[/(\w+)-/, 1])} #{::File.basename($0[/-(.*)/, 1])}" : ::File.basename($0)
-      opts = OptionParser.new 
-      opts.banner = "Usage: #{progname} #{@abstract ? "[command] " : ""}[options]"
+      @opts = OptionParser.new 
+      @opts.banner = "Usage: #{progname} #{@abstract ? "[command] " : ""}[options]"
 
-      opts.separator ""
+      @opts.separator ""
       
       unless @abstract
-        opts.separator "Options:"
+        @opts.separator "Options:"
         
-        opts.on('-v', '--verbose', 'Be verbose')    { self.verbose true }  
-        opts.on('-s [file]', '--spec-file [file]', 'Set the spec file')      { |file| self.spec file.chomp }
-        opts.on('-t', '--test', 'Testing mode')    { self.testing true }
-
-        blk.call(opts, self) if blk
+        @opts.on('-v', '--verbose', 'Be verbose')    { self.verbose true }  
+        @opts.on("--debug", "Debug setting") {self.debug true}
+        @opts.on('-s [file]', '--spec-file [file]', 'Set the spec file')      { |file| self.spec file.chomp }
+        @opts.on('-t', '--test', 'Testing mode')    { self.testing true }
+        
+        blk.call(@opts, self) if blk
       end
       
-      opts.on('-V', '--version', 'Display the version')    { puts @version ; exit 0 }
-      opts.on_tail("-h", "--help", "Show this message") do
-        puts opts
+      @opts.on('-V', '--version', 'Display the version')    { puts @version ; exit 0 }
+      @opts.on_tail("-h", "--help", "Show this message") do
+        puts @opts
         puts @extra_help
         exit
       end
       
-      opts.parse(@arguments.dup)
+      @opts.parse(@arguments.dup)
       
       process_options
       output_options if verbose
-      unless @abstract
+      
+      if @load_pools
         self.loaded_pool load_pool(self.spec || Binary.get_existing_spec_location)
 
         self.loaded_clouds extract_cloud_from_options(self)
         self.loaded_pools extract_pool_from_options(self)
 
         reject_junk_options!
-        raise CloudNotFoundException.new("Please specify your cloud with -s, move it to ./pool.spec or in your POOL_SPEC environment variable") unless loaded_clouds && !loaded_clouds.empty?
+        raise CloudNotFoundException.new("Please specify your cloud with -s, move it to ./clouds.pool or in your POOL_SPEC environment variable") unless loaded_clouds && !loaded_clouds.empty?
         loaded_pools.each do |pl|
           pl.configure(self.options)
         end

data/lib/poolparty/helpers/ruberl.rb

@@ -0,0 +1,33 @@
+require "socket"
+module PoolParty
+  module Ruberl
+    class Base
+      attr_accessor :host, :port
+      def initialize(host="localhost", port=7050)
+        @host = host
+        @port = port
+      end
+      def with_socket(&block)
+        begin
+          socket = TCPSocket.open(@host, @port)
+          out = yield(socket)
+          socket.close
+          out
+        rescue Exception => e
+        end
+      end
+      def messenger_send!(msg="get_current_load cpu")
+        with_socket do |sock|
+          sock.send(msg, 0)
+          @str = sock.recv(2000)      
+        end
+        @str
+      end
+      def messenger_cast!(msg="force_reconfig")
+        with_socket do |sock|
+          sock.send(msg, 0)
+        end
+      end
+    end
+  end
+end

data/lib/poolparty/modules/cloud_dsl.rb

@@ -5,8 +5,9 @@ module PoolParty
       ebs_volume_id id
       ebs_volume_mount_point loc
       ebs_volume_device "/dev/#{id.sanitize}"
-      has_directory(:name => loc)
+            
       has_mount(:name => loc, :device => ebs_volume_device)
+      has_directory(:name => loc)
     end
     
   end

data/lib/poolparty/modules/cloud_resourcer.rb

@@ -4,7 +4,7 @@ module PoolParty
   module CloudResourcer
     
     def plugin_directory(*args)
-      args = ["/plugins"] if args.empty?
+      args = ["#{::File.expand_path(Dir.pwd)}/plugins"] if args.empty?
       args.each {|arg| Dir["#{arg}/*/*.rb"].each {|f| require f rescue "" }}
     end
     
@@ -23,11 +23,25 @@ module PoolParty
       self.run_in_context @stored_block if @stored_block
     end
     
-    # Set instances with a range
-    def instances(arg)
-      if arg.is_a?(Range)
+    # Set instances with a range or a number
+    def instances(arg)      
+      case arg
+      when Range
         minimum_instances arg.first
         maximum_instances arg.last
+      when Fixnum
+        minimum_instances arg
+        maximum_instances arg
+      else
+        raise SpecException.new("Don't know how to handle instances cloud input #{arg}")
+      end
+    end
+    
+    def setup_dev
+      unless ::File.exists?("#{full_keypair_basename_path}.pub")
+        cmd = "scp #{ssh_array.join(" ")} #{Base.user}@#{master.ip}:.ssh/authorized_keys #{full_keypair_basename_path}.pub"
+        vputs "Running #{cmd}"
+        Kernel.system(cmd)
       end
     end
     
@@ -38,6 +52,14 @@ module PoolParty
         ::File.expand_path(keypair_path)
       end
     end
+    def full_pub_keypair_path
+      @full_pub_keypair_path ||= ::File.expand_path("#{full_keypair_basename_path}.pub")
+    end
+    def full_keypair_basename_path
+      dir = ::File.dirname(full_keypair_path)
+      basename = ::File.basename(full_keypair_path, ::File.extname(full_keypair_path))
+      ::File.join(dir, basename)
+    end
     
     def keypair_path
       keypair_paths.each do |path|
@@ -49,6 +71,8 @@ module PoolParty
       return nil
     end
     
+    # The keypair name can be one name or another including id_rsa or not
+    # So let's get the name that exists as a keypair
     def full_keypair_name
       keypair_paths.each do |path|
         possible_keypair_basenames.each do |base|
@@ -120,8 +144,8 @@ module PoolParty
       @plugin_store ||= []
     end
     
-    def realize_plugins!
-      plugin_store.each {|plugin| plugin.realize! if plugin }
+    def realize_plugins!(force=false)
+      plugin_store.each {|plugin| plugin.realize!(force) if plugin }
     end
     
     def plugin_store

data/lib/poolparty/modules/configurable.rb

@@ -1,3 +1,5 @@
+#TODO: rdoc: this  defines methods on poolparty objects from a passed hash of options.
+# For example, this is how instance.minimum_runtime is set.  See base.rb line 12 for example of default options that are added as methods in this way. 
 module PoolParty
   module Configurable
     module ClassMethods      

data/lib/poolparty/modules/file_writer.rb

@@ -2,8 +2,8 @@ module PoolParty
   module FileWriter
     def copy_file_to_storage_directory(file, preceded="")
       make_base_directory
-      path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )      
-      FileUtils.cp file, path unless file == path || ::File.exists?(path)
+      path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
+      ::FileUtils.cp file, path unless file == path || ::File.file?(path)
     end
     def cleanup_storage_directory
       Dir["#{Base.storage_directory}/**/*"].each do |f|
@@ -26,11 +26,11 @@ module PoolParty
       path = ::File.join( Base.storage_directory, dirname )
       make_base_path path
     end
-    def write_to_file_in_storage_directory(file, str, preceded="", &block)
+    def write_to_file_in_storage_directory(file, str="", preceded="", &block)
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       write_to_file(path, str, &block)
     end
-    def write_to_file(file, str, preceded="", &block)
+    def write_to_file(file, str="", preceded="", &block)
       path = ::File.join( Base.storage_directory, preceded, ::File.basename(file) )
       make_base_path( Base.storage_directory )
       ::File.open(path, "w+") do |f|

data/lib/poolparty/modules/resourcing_dsl.rb

@@ -10,12 +10,12 @@ module PoolParty
       str ? options.append!(:require => send_if_method(str)) : options[:require]
     end 
     def ensures(str="running")
-      if %w(absent running).map {|a| self.send a.to_sym}.include?(str)
+      # if %w(absent running).map {|a| self.send a.to_sym}.include?(str)
         str == "absent" ? is_absent : is_present
-      else
-        options.append!(:ensure => str)
-      end
-      str
+      # else
+        # options.append!(:ensure => str)
+      # end
+      # str
     end
     # Allows us to send an ensure to ensure the presence of a resource
     def is_present(*args)

data/lib/poolparty/monitors/base_monitor.rb

@@ -29,7 +29,7 @@
   Notice that at the end, you must call register_monitor :monitorname. This will tell your cloud
   that it can monitor it with this monitor.
 =end
-require "#{::File.dirname(__FILE__)}/../pool/base"
+require "#{::File.dirname(__FILE__)}/../poolparty/base"
 
 module PoolParty
   module Monitors    

data/lib/poolparty/monitors/monitors/cpu_monitor.rb

@@ -4,8 +4,8 @@ module PoolParty
     class CpuMonitor < BaseMonitor
       
       def run
-        str = %x[uptime]
-        str.split(/\s+/)[-1].to_f rescue 0.0
+        stdin, stdout, stderr = Open3.popen3('uptime')
+        stdout.split(/\s+/)[-1].to_f rescue 0.0
       end
             
     end

data/lib/poolparty/net/messenger.rb

@@ -16,7 +16,7 @@ module PoolParty
     end
     # TODO: Fix cookie setting
     def self.erl_command(hostname, extra="", min_ports=7000, max_ports=7050)
-      command_line_opts = "-pa #{append_dir}/ebin -kernel inet_dist_listen_min #{min_ports} inet_dist_listen_max #{max_ports} -sname #{hostname} -setcookie poolparty"
+      command_line_opts = "-pa #{append_dir}/ebin -kernel inet_dist_listen_min #{min_ports} inet_dist_listen_max #{max_ports} -sname #{hostname}"
       
       "erl #{command_line_opts} #{extra}"
     end

data/lib/poolparty/net/remote.rb

@@ -8,8 +8,8 @@ module PoolParty
     
     def using(t)
       @cloud = self
-      if available_bases.include?(t.to_sym)
-        unless using_remoter? || t.nil?
+      if t && available_bases.include?(t.to_sym)
+        unless using_remoter?
           self.class.send :attr_reader, :remote_base
           self.class.send :attr_reader, :parent_cloud
           mod = "#{t}".preserved_module_constant

data/lib/poolparty/net/remote_bases/ec2.rb

@@ -29,7 +29,7 @@ begin
           :minCount => 1,
           :maxCount => num,
           :key_name => (keypair || Base.keypair),
-          :availability_zone => nil,
+          :availability_zone => (availabilty_zone || Base.availabilty_zone),
           :instance_type => "#{size || Base.size}",
           :group_id => ["#{security_group || Base.security_group}"])
         begin
@@ -71,18 +71,24 @@ begin
         EC2ResponseObject.get_descriptions(ec2.describe_instances)
       end
 
-      def after_launch_master(instance=nil)
+      def after_launch_master(inst=nil)
+        instance = master
+        vputs "Running tasks after launching the master"
         begin
-          when_no_pending_instances do
+          # when_no_pending_instances do
             if instance
-              ec2.associate_address(:instance_id => instance.instanceId, :public_ip => set_master_ip_to) if set_master_ip_to
-              ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instanceId, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
+              ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instance_id, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
+              # Let's associate the address LAST so that we can still connect to the instance
+              # for the other tasks here              
+              ec2.associate_address(:instance_id => instance.instance_id, :public_ip => set_master_ip_to) if set_master_ip_to              
+              reset_remoter_base!
             end
-          end
+          # end
         rescue Exception => e        
+          vputs "Error in after_launch_master: #{e}"
         end
         reset_remoter_base!
-        when_all_assigned_ips {wait "2.seconds"}
+        when_all_assigned_ips {wait "5.seconds"}
       end
 
       # Help create a keypair for the cloud
@@ -108,23 +114,57 @@ begin
                                 :secret_access_key => (secret_access_key || Base.secret_access_key)
                               )
       end
+      
+      def before_configuration_tasks
+        if has_cert_and_key?
+          # copy_file_to_storage_directory(pub_key)
+          # copy_file_to_storage_directory(private_key)
+        end
+      end
+      def has_cert_and_key?
+        pub_key && private_key
+      end
+      # The keys are used only for puppet certificates
+      # and are only used for EC2.
+      # Public key 
+      def pub_key
+        @pub_key ||= ENV["EC2_CERT"] ? ENV["EC2_CERT"] : nil
+      end
+      # Private key
+      def private_key
+        @private_key ||= ENV["EC2_PRIVATE_KEY"] ? ENV["EC2_PRIVATE_KEY"] : nil
+      end
+      
+      def custom_minimum_runnable_options
+        [:ami, :availabilty_zone, :security_group]
+      end
 
-      # Callback
+      # Hook
+      #TODO#: Change this so they match with the cap tasks
       def custom_install_tasks_for(o)
-        [
+        arr = if has_cert_and_key?
+          [ 
+            # "mv #{::File.basename(pub_key)} #{Base.base_config_directory}/ssl/public_keys/#{o.name}.pem", 
+            # "mv #{::File.basename(private_key)} #{Base.base_config_directory}/ssl/private_keys/#{o.name}.pem"
+          ]
+          else 
+            []
+          end
+        arr << [
           "# ec2 installation tasks",
           "# Set hostname",
           # "if [ -z $(grep -v '#' /etc/hosts | grep '#{o.name}') ]; then echo \"$(curl http://169.254.169.254/latest/meta-data/public-ipv4) #{o.name}\" >> /etc/hosts; fi",
           "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
           "hostname #{o.name}",
           "echo #{o.name} > /etc/hostname",
-          "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1 && gem install -y --no-ri --no-rdoc amazon-ec2.gem 2>&1"
+          "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
+          "/usr/bin/gem install -y --no-ri --no-rdoc amazon-ec2.gem 2>&1",
         ]
+        []
       end
 
       def custom_configure_tasks_for(o)
         [
-          "# ec2 configuration"
         ]
       end
 

data/lib/poolparty/net/remote_instance.rb

@@ -8,17 +8,21 @@ module PoolParty
       include Configurable
       include CloudResourcer
       
-      def initialize(opts, parent=nil)        
+      def initialize(opts, parent=self)        
         run_setup(parent)
-                
+
         set_vars_from_options(parent.options) if parent && parent.respond_to?(:options)
         set_vars_from_options(opts) unless opts.nil? || opts.empty?
         
         on_init
       end
       
+      def elapsed_runtime
+        Time.now.to_i - launching_time.to_time.to_i
+      end
+      
       # Callback
-      def on_init        
+      def on_init
       end
       
       # Is this remote instance the master?