auser-poolparty 0.2.2 → 0.2.3

data/lib/poolparty/modules/pretty_printer.rb

@@ -23,7 +23,8 @@ module PoolParty
     # Do not print if the option is nil or empty.
     # Also, don't show the option if the option is empty or the default option on the cloud    
     def pretty_options(prev, o)
-      print_options = (o.respond_to?(:parent) && o.parent) ? 
+      return "" unless o.respond_to?(:options)
+      print_options = (o.respond_to?(:parent) && o.parent && o.parent.respond_to?(:options)) ? 
         (o.options.delete_if {|k,v| o.parent.options.has_key?(k) && o.parent.options[k] == o.options[k] && !o.options[k].nil? } ) : 
         o.options
       print_options = print_options.map {|k,v| [k, o.send(k.to_sym).to_s] }.inject({}) { |r,e| r[e[0]] = e[1] unless o.class.default_options[e[0]] == e[1] || e[1].nil? || e[1].empty?; r }

data/lib/poolparty/net/remote.rb

@@ -12,7 +12,7 @@ module PoolParty
           self.class.send :attr_reader, :remote_base
           mod = "#{t}".preserved_module_constant
           
-          mod.send :include, PoolParty::Remote::RemoterBase                    
+          mod.send :include, PoolParty::Remote::RemoterBase
           self.class.send :include, mod
           self.extend mod
                     

data/lib/poolparty/net/remote_bases/ec2.rb

@@ -1,4 +1,6 @@
-require "ec2"
+require 'rubygems'
+require 'EC2'
+
 class String
   def convert_from_ec2_to_ip
     self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
@@ -6,15 +8,15 @@ class String
 end
 module PoolParty
   module Ec2
-    def launch_new_instance!
+    def launch_new_instance!(num=1)
       instance = ec2.run_instances(
-        :image_id => self.respond_to?(:ami) ? ami : Base.ami,
+        :image_id => (ami || Base.ami),
         :user_data => "",
         :minCount => 1,
-        :maxCount => 1,
-        :key_name => "#{self.respond_to?(:keypair) ? keypair : Base.keypair}",
+        :maxCount => num,
+        :key_name => (keypair || Base.keypair),
         :availability_zone => nil,
-        :size => "#{self.respond_to?(:size) ? size : Base.size}")
+        :size => "#{size || Base.size}")
       begin
         item = instance#.instancesSet.item
         EC2ResponseObject.get_hash_from_response(item)
@@ -34,7 +36,7 @@ module PoolParty
         @id = 0
         @describe_instances = get_instances_description.each_with_index do |h,i|
           if h[:status] == "running"
-            @name = "node#{@id}"
+            @name = @id == 0 ? "master" : "node#{@id}"
             @id += 1
           else
             @name = "#{h[:status]}_node#{i}"
@@ -45,7 +47,6 @@ module PoolParty
             :ip => h[:ip].convert_from_ec2_to_ip
           })
         end
-        @describe_instances.first[:name] = "master" unless @describe_instances.empty?
       end
       @describe_instances
     end
@@ -58,8 +59,10 @@ module PoolParty
     # This is a helper to create the keypair and add them to the cloud for you
     def create_keypair
       return false unless keypair
-      FileUtils.mkdir_p ::File.dirname(new_keypair_path) unless ::File.directory?(::File.dirname(new_keypair_path))
-      Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+      unless ::File.exists?( new_keypair_path )
+        FileUtils.mkdir_p ::File.dirname( new_keypair_path )
+        Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+      end
     end
     # EC2 connections
     def ec2

data/lib/poolparty/net/remote_instance.rb

@@ -9,8 +9,9 @@ module PoolParty
       include CloudResourcer
             
       def initialize(opts, parent=self)
-        set_parent(parent) if parent
-        set_vars_from_options(opts) unless opts.empty?        
+        @parent = parent
+        set_vars_from_options(parent.options) if parent && parent.respond_to?(:options)
+        set_vars_from_options(opts) unless opts.nil? || opts.empty?
         on_init
       end
       

data/lib/poolparty/net/remoter.rb

@@ -46,7 +46,7 @@ module PoolParty
           out = list_from_remote(:cache => true)
         end
         return out
-      end
+      end      
       # List the instances that are known from the remoter_base
       # Create a RemoteInstance for each of the instances from the hash
       # returned by the list of instances, write them to the cached file
@@ -87,14 +87,15 @@ module PoolParty
         ]
       end
       
-      # More functional methods
+      # List calculation methods
+      # 
       # Are the minimum number of instances running?
       def minimum_number_of_instances_are_running?
-        list_of_running_instances.size >= minimum_instances
+        list_of_running_instances.size >= minimum_instances.to_i
       end
       # Can we shutdown an instance?
       def can_shutdown_an_instance?
-        list_of_running_instances.size > minimum_instances
+        list_of_running_instances.size > minimum_instances.to_i
       end
       # Request to launch a number of instances
       def request_launch_new_instances(num=1)
@@ -113,38 +114,52 @@ module PoolParty
       end
       # Are the maximum number of instances running?
       def maximum_number_of_instances_are_not_running?
-        list_of_running_instances.size < maximum_instances
+        list_of_running_instances.size < maximum_instances.to_i
       end
       # Launch new instance while waiting for the number of pending instances
       #  to be zero before actually launching. This ensures that we only
       #  launch one instance at a time
-      def request_launch_one_instance_at_a_time        
-        when_no_pending_instances do
-          launch_new_instance!
-        end
+      def request_launch_one_instance_at_a_time
+        when_no_pending_instances { launch_new_instance! }
       end
       # A convenience method for waiting until there are no more
       # pending instances and then running the block
       def when_no_pending_instances(&block)
         reset!
-        if list_of_pending_instances.size > 0
+        if list_of_pending_instances.size == 0
+          block.call if block
+        else
           wait "5.seconds"
           when_no_pending_instances(&block)
-        else
-          block.call if block
         end
       end
       
       # This will launch the minimum_instances if the minimum number of instances are not running
       # If the minimum number of instances are not running and if we can start a new instance
-      def launch_minimum_number_of_instances
-        if can_start_a_new_instance?
-          while !minimum_number_of_instances_are_running?
-            request_launch_one_instance_at_a_time
-            wait "5.seconds"
-          end
+      def launch_minimum_number_of_instances        
+        if can_start_a_new_instance? && !minimum_number_of_instances_are_running?         
+          list_of_pending_instances.size == 0 ? request_launch_one_instance_at_a_time : wait("5.seconds")
+          reset!
+          launch_minimum_number_of_instances unless minimum_number_of_instances_are_running?
         end
       end
+      # Launch the master and let the master handle the starting of the cloud
+      # We should only launch an instance if there are no pending instances, in the case 
+      # that the master has launched, but is still pending
+      # and if the master is not running AND we can start a new instance
+      # Then wait for the master to launch
+      def launch_and_configure_master!(testing=false)
+        request_launch_new_instances(1) if list_of_pending_instances.size.zero? && can_start_a_new_instance? && !is_master_running?
+        
+        when_no_pending_instances do
+          wait "5.seconds"
+          Provisioner.provision_master(self, testing)
+        end
+        
+      end
+      def is_master_running?
+        !list_of_running_instances.select {|a| a.name == "master"}.first.nil?
+      end
       # Stub method for the time being to handle expansion of the cloud
       def should_expand_cloud?(force=false)
         force || false

data/lib/poolparty/net/remoter_base.rb

@@ -54,7 +54,7 @@ module PoolParty
         list.reject {|i| true if !i.terminating? }
       end
       # Get the instances that are non-master instances
-      def nonmaster_nonterminated_instances(list = remote_instances_list)
+      def nonmaster_nonterminated_instances(list = list_of_nonterminated_instances)
         list_of_nonterminated_instances.reject {|i| i.master? }
       end
       # list all the nonterminated instances
@@ -73,7 +73,7 @@ module PoolParty
       # If no keypair is passed, select them all
       def list_of_instances(keyp=nil)
         key = keyp ? keyp : keypair
-        describe_instances.select {|a| key ? a[:keypair] == key : true }
+        describe_instances.select {|a| key ? a[:keypair] == key : true } if describe_instances
       end
       # Instances
       # Get the master from the cloud
@@ -87,7 +87,7 @@ module PoolParty
       end
       # Reset the cache of descriptions
       def reset!
-      end      
+      end
       def self.included(other)
         PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)
       end

data/lib/poolparty/plugins/gem_package.rb

@@ -1,38 +1,15 @@
 module PoolParty    
   class Gem
         
-    define_resource(:gem_package) do
+    virtual_resource(:gempackage) do
       
-      def has_gem_package(opts={})
-        call_function <<-EOE
-gem_package { "#{opts[:package] || opts[:name]}":
-  source => "#{opts[:source] || "http://gems.github.com" }",
-  package => "#{opts[:package] || opts[:name]}"
-}
-        EOE
+      def loaded
+        has_exec(:name => "gem-package-#{name}", :cwd => "/tmp", :path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin") do
+          command "gem install --no-ri --no-rdoc #{"--version \"#{version}\"" if respond_to?(:version)} #{"--source \"#{source}\"" if respond_to?(:source)} #{@parent.name}"
+          ifnot "gem list --local #{@parent.name} | grep #{@parent.name} #{"| grep #{version}" if respond_to?(:version)}"
+        end
       end
       
-      custom_function <<-EOF
-      define gem_package ($source = "http://gems", $version, $package=false) {
-        include ruby
-
-        if $version {
-          exec { "gem-package-$package":
-            path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin",
-            cwd => "/tmp",
-            command => "gem install --source $source --version \"$version\" $package",
-            unless => "gem list --local $package | grep \"$package\" | grep \"$version\""
-          }
-        } else {
-          exec { "gem-package-$package":
-            path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin",
-            cwd => "/tmp",
-            command => "gem install --source $source $package",
-            unless => "gem list --local $package | grep \"$package\"
-          }
-        }
-      }
-      EOF
     end
     
   end

data/lib/poolparty/plugins/git.rb

@@ -0,0 +1,22 @@
+module PoolParty    
+  class Git
+        
+    virtual_resource(:git) do
+      
+      def loaded
+        
+        has_directory(:name => "#{name}", :path => "#{path}", :user => "#{user || Base.user}")
+        
+        exec({:name => "git-#{name}", :command => (user ? "git clone #{user}@#{source}" : "git clone #{source}")}) do
+          cwd "#{path}"
+          requires "File[#{name}]"
+        end
+        
+        exec(:name => "git-update-#{name}", :cwd => "#{path}") do
+          requires "Exec['git-#{name}']"
+        end        
+      end
+    end
+    
+  end
+end

data/lib/poolparty/pool/base.rb

@@ -20,21 +20,19 @@ module PoolParty
       :base_config_directory => "/etc/poolparty",
       :template_directory => "tmp/templates",
       :template_path => "/var/lib/puppet/templates",
+      :default_specfile_name => "pool.spec",
       :port => "80",
       :forwarding_port => "8080",
       :proxy_mode => "http",
       :pool_logger_location => File.join(Dir.pwd, "logs"),
       # EC2 Options
-      :ami => "ami-4bb05422" 
+      :ami => "ami-1cd73375" 
     })
         
     # Class methods
     class << self
-      def actionable_default_options
-        default_options.merge!({:access_key => self.access_key,:secret_access_key => secret_access_key})
-      end
       def options(h={})
-        @options ||= actionable_default_options.merge(h)
+        @options ||= default_options.merge(h)
       end
       # Get the access_key
       def access_key
@@ -46,6 +44,8 @@ module PoolParty
       def load_keys_from_file
         @keys ||= get_working_key_file_locations ? YAML::load( open(get_working_key_file_locations).read ) : {}
       end
+      # Store the keys in a yaml format to give the master access
+      # So that the master has access to the files
       def store_keys_in_file
         unless access_key.nil? || secret_access_key.nil?
           write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
@@ -62,8 +62,9 @@ module PoolParty
       # Expected places for the instances.list to be located at on the machine
       def key_file_locations
         [
-          "#{Base.storage_directory}/.ppkeys",
+          ".ppkeys",
           "#{Base.base_config_directory}/.ppkeys",
+          "#{Base.storage_directory}/ppkeys",          
           "~/.ppkeys",
           "ppkeys"
         ]

data/lib/poolparty/pool/cloud.rb

@@ -23,7 +23,7 @@ module PoolParty
       include PrettyPrinter
       include Configurable
       include CloudResourcer
-      extend CloudResourcer
+      # extend CloudResourcer
       # Net methods
       include PoolParty::Remote::RemoterBase
       include Remote
@@ -40,11 +40,12 @@ module PoolParty
       })
       
       def initialize(name, parent, &block)
-        @name = name        
+        @name = name
+        # store_block(&block)
         set_parent(parent) if parent
-        self.instance_eval &block if block_given?
+        self.instance_eval &block if block
         # this can be overridden in the spec, but ec2 is the default
-        using :ec2
+        self.using :ec2
       end
                         
       # Keypairs
@@ -60,13 +61,34 @@ module PoolParty
         options[:keypair] = args.length > 0 ? args[0] : "#{@parent.respond_to?(:name) ? @parent.name : ""}_#{@name}"
       end
       
+      # Prepare to send the new configuration to the instances
+      # First, let's make sure that our base directory is made
+      # Then copy the templates that have no other reference in
+      # a spec file. Make sure the keys are stored in a file
+      # For the master to have access to them
+      # Then, send the saved containing cloud instances to give the 
+      # remote master access to the cloud options that are required
+      # for the master to run checks
       def prepare_to_configuration        
         # clear_base_directory
         make_base_directory
         copy_misc_templates
         Base.store_keys_in_file
+        Script.save!
+        copy_ssh_key # not my favorite...
       end
       
+      def copy_ssh_key
+        copy_file_to_storage_directory(full_keypair_path)
+      end
+      
+      # Build the new poolparty manifest
+      # Wrapping all of these requirements into the one 
+      # poolparty class.
+      # 
+      # TODO: Consider the benefits of moving all the manifest
+      # classes to separate files and keeping the containing
+      # references in the include
       def build_and_store_new_config_file
         @manifest = build_manifest
         config_file = ::File.join(Base.storage_directory, "poolparty.pp")
@@ -85,7 +107,7 @@ module PoolParty
       
       # Configuration files
       def build_manifest
-        reset_resources!        
+        reset_resources!
         add_poolparty_base_requirements
         
         returning Array.new do |str|
@@ -105,6 +127,13 @@ module PoolParty
         end.join("\n")
       end
       
+      # To allow the remote instances to do their job,
+      # they need a few options to run, these are the required options
+      # to be saved on the remote "master" machine
+      def minimum_runnable_options
+        [:keypair, :minimum_instances, :maximum_instances]
+      end
+      
       # Add all the poolparty requirements here
       # NOTE: These are written as plugins in the lib/poolparty/base_packages directory
       # for examples. 

data/lib/poolparty/pool/custom_resource.rb

@@ -20,9 +20,9 @@ module PoolParty
                 
     # Resources for function call
     class CallFunction < Resource
-      def initialize(str="", opts={}, &block)
+      def initialize(str="", opts={}, parent=self, &block)
         @str = str
-        super(opts, &block)
+        super(opts, parent, &block)
       end
       def to_string(prev="")
         returning Array.new do |arr|
@@ -32,9 +32,9 @@ module PoolParty
     end
         
     class CustomResource < Resource
-      def initialize(name=:custom_method, opts={}, &block)
+      def initialize(name=:custom_method, opts={}, parent=self, &block)
         @name = name
-        super(opts, &block)
+        super(opts, parent, &block)
       end
       
       def self.inherited(subclass)

data/lib/poolparty/pool/plugin.rb

@@ -6,7 +6,6 @@ module PoolParty
       include Configurable
       include CloudResourcer
       include Resources
-      extend Resources
       
       attr_accessor :parent
       class_inheritable_accessor :name
@@ -22,19 +21,19 @@ module PoolParty
       def enable
       end
             
-      def method_missing(m, *args, &block)
-        if block_given?
-          (args[0].class == self.class) ? args[0].instance_eval(&block) : super
-        elsif parent && parent.respond_to?(m)
-          parent.send m, *args, &block
-        else
-          get_from_options(m, *args)
-        end
-      end
-
-      def get_from_options(m, *args)
-        args.empty? ? options[m] : options[m] = args[0]
-      end
+      # def method_missing(m, *args, &block)
+      #   if block_given?
+      #     (args[0].class == self.class) ? args[0].instance_eval(&block) : super
+      #   elsif parent && parent.respond_to?(m)
+      #     parent.send m, *args, &block
+      #   else
+      #     get_from_options(m, *args)
+      #   end
+      # end
+      # 
+      # def get_from_options(m, *args)
+      #   args.empty? ? options[m] : options[m] = args[0]
+      # end
       
     end