auser-poolparty 1.0.0 → 1.1.1

data/lib/poolparty/net/{remote_bases → remoter_bases/ec2}/ec2.rb

@@ -5,7 +5,7 @@
   cluster. 
 =end
 require "date"
-require "#{::File.dirname(__FILE__)}/ec2/ec2_response_object"
+require "#{::File.dirname(__FILE__)}/ec2_response_object"
 
 begin
   require 'EC2'
@@ -18,53 +18,58 @@ a different remoter base, specify it with:
 
 using :remoter_name
 
-in your config file, otherwise, to continue install amazon-ec2 with
+in your config file, otherwise, to continue install grempe-amazon-ec2 with
 
-gem install amazon-ec2
+sudo gem install grempe-amazon-ec2 --source http://gems.github.com
 EOM
 end
   
-  class String
-    def convert_from_ec2_to_ip
-      self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
-    end
-    def parse_datetime
-      DateTime.parse( self.chomp ) rescue self
-    end
+class String
+  def convert_from_ec2_to_ip
+    self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
   end
-  module PoolParty
-    module Ec2
-      include PoolParty::Remote::RemoterBase
-      
-      def launch_new_instance!(num=1)
-        instance = ec2.run_instances(
-          :image_id => (ami || Base.ami),
-          :user_data => "",
+  def parse_datetime
+    DateTime.parse( self.chomp ) rescue self
+  end
+end
+  
+module PoolParty    
+  module Remote
+    class Ec2 < Remote::RemoterBase
+
+      def self.launch_new_instance!(o = options)
+        raise "You must pass a keypair to launch an instance, or else you wont be able to login. options = #{o.inspect}" if !o[:keypair]
+        instance = ec2(o).run_instances(
+          :image_id => o[:ami],
+          :user_data => o[:user_data],
           :minCount => 1,
-          :maxCount => num,
-          :key_name => (keypair || Base.keypair),
-          :availability_zone => (availabilty_zone || Base.availabilty_zone),
-          :instance_type => "#{size || Base.size}",
-          :group_id => ["#{security_group || Base.security_group}"])
+          :maxCount => o[:num],
+          :key_name => o[:keypair],
+          :availability_zone => o[:availabilty_zone],
+          :instance_type => o[:size],
+          :group_id => o[:security_group])
+
         begin
-          h = EC2ResponseObject.get_hash_from_response(instance)
+          h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
           #h = instance.instancesSet.item.first
         rescue Exception => e
-          h = instance
+          h = EC2ResponseObject.get_hash_from_response(instance) rescue instance
+          # h = instance
         end
         h
       end
       # Terminate an instance by id
-      def terminate_instance!(instance_id=nil)
-        ec2.terminate_instances(:instance_id => instance_id)
+      def self.terminate_instance!(o={})  #NOTE: maybe we should not allow this command wihtout an instance_idˇ
+        ec2(o).terminate_instances(:instance_id => o[:instance_id])
       end
       # Describe an instance's status
-      def describe_instance(id=nil)
-        describe_instances.select {|a| a[:name] == id}[0] rescue nil
+      def self.describe_instance(o={})
+        return describe_instances.first if o[:instance_id].nil?
+        describe_instances.detect {|a| a[:name] == o[:instance_id] || a[:ip] == o[:instance_id] || a[:instance_id] == o[:instance_id] }
       end
-      def describe_instances
+      def self.describe_instances(o={})
         id = 0
-        get_instances_description.each_with_index do |h,i|
+        get_instances_description(o).each_with_index do |h,i|
           if h[:status] == "running"
             inst_name = id == 0 ? "master" : "node#{id}"
             id += 1
@@ -75,14 +80,34 @@ end
             :name => inst_name,
             :hostname => h[:ip],
             :ip => h[:ip].convert_from_ec2_to_ip,
-            :index => i,
+            :index => i,  #TODO MF get the instance id from the aws result instead
             :launching_time => (h[:launching_time])
           })
         end.sort {|a,b| a[:index] <=> b[:index] }
       end
-      # Get the s3 description for the response in a hash format
-      def get_instances_description
-        EC2ResponseObject.get_descriptions(ec2.describe_instances)
+      
+      def self.ec2(o={})
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
+                                :secret_access_key => o[:secret_access_key]
+                              )
+      end
+      # Get the ec2 description for the response in a hash format
+      def self.get_instances_description(o={})
+        EC2ResponseObject.get_descriptions(ec2(o).describe_instances)
+      end
+      def get_descriptions(o={})
+        self.class.get_descriptions(o)
+      end
+      
+      # Class method helpers
+      def self.aws_keys
+        unless @access_key && @secret_access_key          
+          aws_keys = {}
+          aws_keys = YAML::load( File.open('/etc/poolparty/aws_keys.yml') ) rescue 'No aws_keys.yml file.   Will try to use enviornment variables'
+          @access_key ||= aws_keys[:access_key] || ENV['AMAZON_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY']
+          @secret_access_key ||= aws_keys[:secret_access_key] || ENV['AMAZON_SECRET_ACCESS_KEY'] || ENV['AWS_SECRET_ACCESS_KEY']
+        end
+        [@access_key, @secret_access_key]
       end
 
       def after_launch_master(inst=nil)
@@ -104,8 +129,9 @@ end
         reset_remoter_base!
         when_all_assigned_ips {wait "5.seconds"}
       end
-      
+    
       # Attach a volume to the instance
+      # DEPRECATE this relies on master.  master will be removed in next major release.  This method will be in ec2_remote_instance instead, or require an instance id
       def attach_volume(instance=nil)
         if ebs_volume_id
           vputs "Attaching volume #{ebs_volume_id} to the master at #{ebs_volume_device}"
@@ -114,6 +140,7 @@ end
         end
       end
       # Associate an address with the instance using ec2
+      # DEPRECATE relies on master
       def associate_address(instance=nil)
         if set_master_ip_to
           dputs "Associating master with #{set_master_ip_to}"
@@ -132,33 +159,31 @@ end
           Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
         end
       end
-      
+    
       # wrapper for remote base to perform a snapshot backup for the ebs volume
       def create_snapshot
         return nil if ebs_volume_id.nil?
         ec2.create_snapshot(:volume_id => ebs_volume_id)
       end
-      
+    
       # EC2 connections
-      def ec2
-        @ec2 ||= EC2::Base.new( :access_key_id => (access_key || Base.access_key), 
-                                :secret_access_key => (secret_access_key || Base.secret_access_key)
-                              )
+      def ec2(o={})
+        @ec2 ||= self.class.ec2(o)
       end
-      
+    
       # These are tasks that run before the configuration runs
       def before_configuration_tasks
         if set_master_ip_to && master.ip && master.ip.to_s != set_master_ip_to.to_s
           associate_address(master)
           reset_remoter_base!
-        
+      
           when_no_pending_instances do
             when_all_assigned_ips do
               vputs "Associated master with #{set_master_ip_to}"
             end
           end
         end
-        
+      
       end
       def has_cert_and_key?
         pub_key && private_key
@@ -173,7 +198,7 @@ end
       def private_key
         @private_key ||= ENV["EC2_PRIVATE_KEY"] ? ENV["EC2_PRIVATE_KEY"] : nil
       end
-      
+    
       def custom_minimum_runnable_options
         [:ami, :availabilty_zone, :security_group]
       end
@@ -188,11 +213,11 @@ end
           "echo #{o.name} > /etc/hostname"
         ]
       end
-      
+    
       def after_install_tasks_for(o)
         [
-          "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
-          "/usr/bin/gem install --no-ri --no-rdoc amazon-ec2.gem 2>&1"
+          # "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
+          # "/usr/bin/gem install --no-ri --no-rdoc amazon-ec2.gem 2>&1"
         ]
       end
 
@@ -203,7 +228,8 @@ end
 
       def reset_base!
         @describe_instances = @cached_descriptions = nil
-      end
+      end      
     end
-    register_remote_base :Ec2
-  end
+        
+  end
+end

data/lib/poolparty/net/remoter_bases/ec2/ec2_remote_instance.rb

@@ -0,0 +1,53 @@
+module PoolParty  
+  module Remote
+    class Ec2RemoteInstance < RemoteInstance
+      include Dslify
+      
+      attr_reader :my_cloud, :uniquely_identifiable_by, :found_at
+
+      
+      # A new instance will be created from the passed in hash.  
+      # This hash of passed in values will be converted to methods on this instance.
+      # The parent clouds describe_instances list will be searched for the first one matching any of this instance's provided unique identifiers.
+      # If an instance is found, this instance's properties will be set to the properties provided
+      # If the found instance has properties of the same key as the provided options, the found instance's values will override the passed in options
+      def initialize(opts={}, prnt=Ec2.new)
+        @uniquely_identifiable_by = [:ip, :name, :dns_name, :instance_id]
+        @original_options = opts
+        @my_cloud = prnt
+        super(opts, prnt)
+        find_myself(@uniquely_identifiable_by && opts.keys) if prnt.respond_to?(:describe_instances)
+      end
+   
+      # Search the clouds describe_instances list for the first match on one of this nodes unique identifiers
+      def find_myself(unique_identifiers = @original_options.keys)
+        description_hash = @my_cloud.describe_instances.detect do|node|
+          unique_identifiers.detect{|identifier_key| node[identifier_key] == options[identifier_key]
+            }
+        end
+        return nil if description_hash.blank?
+        @found_at = Time.now
+        self.set_vars_from_options(description_hash)
+        self
+      end
+      
+      # Is this instance running?
+      def running?
+        !(status =~ /running/).nil?
+      end
+      # Is this instance pending?
+      def pending?
+        !(status =~ /pending/).nil?
+      end
+      # Is this instance terminating?
+      def terminating?
+        !(status =~ /shutting/).nil?
+      end
+      # Has this instance been terminated?
+      def terminated?
+        !(status =~ /terminated/).nil?
+      end
+      
+    end
+  end
+end

data/lib/poolparty/net/{remote_bases → remoter_bases}/ec2/ec2_response_object.rb

@@ -1,3 +1,5 @@
+# Convenience class to convert standard amazon-ec2 responses from their camel cased style to a hash using underscore style.
+# For example: instanceId to instance_id
 class EC2ResponseObject
   def self.get_descriptions(resp)      
     rs = get_instance_from_response(resp)
@@ -48,7 +50,6 @@ class EC2ResponseObject
     group
   end
   def self.get_hash_from_response(resp, group = 'default')
-    begin
       {
         :instance_id => resp.instanceId,
         :name => resp.instanceId, 
@@ -59,8 +60,5 @@ class EC2ResponseObject
         :keypair => resp.keyName,
         :security_group => group
       }        
-    rescue Exception => e
-      nil
-    end      
   end
 end

data/lib/poolparty/plugins/chef.rb

@@ -0,0 +1,168 @@
+require "tempfile"
+module PoolParty
+  class ChefRecipe
+    include Dslify
+  end
+  class Chef
+    
+    plugin :chef do
+      def before_load(o, &block)        
+        bootstrap_gems "chef", "ohai"
+        bootstrap_commands [
+          "mkdir -p /etc/chef/cookbooks /etc/chef/cache"
+        ]
+      end
+      
+      def loaded o={}, &block
+      end
+      
+      def recipe_files
+        @recipe_files ||= []
+      end
+      
+      def basedir
+        @basedir ||= "#{Default.tmp_path}/dr_configure/chef/recipes/main"
+      end
+      
+      def recipe file=nil, o={}, &block        
+        if file
+          ::FileUtils.mkdir_p "#{basedir}/recipes" unless ::File.directory? basedir
+          
+          unless ::File.file?(file)
+            tfile = Tempfile.new("main-poolparty-recipe")
+            tfile << file # copy the string into the temp file
+            file = tfile.path
+          end
+          
+          ::File.cp file, "#{basedir}/recipes/default.rb"
+          
+          templates o[:templates] if o[:templates]
+          
+          recipe_files << basedir
+        # TODO: Enable neat syntax from within poolparty
+        else
+          raise <<-EOR
+            PoolParty currently only supports passing recipes as files. Please specify a file in your chef block and try again"
+          EOR
+        end
+      end
+      
+      def templates templates=[]
+        if templates
+          ::FileUtils.mkdir_p "#{basedir}/templates/default/"
+          templates.each do |f|
+            if ::File.file?(f)
+              ::File.cp f, "#{basedir}/templates/default/#{::File.basename(f)}"
+            elsif ::File.directory?(f)
+              Dir["#{f}/*"].each {|f| ::File.cp f, "#{basedir}/templates/default/#{::File.basename(f)}" }
+            else
+              tfile = Tempfile.new("main-poolparty-recipe")
+              tfile << f # copy the string into the temp file
+              ::File.cp tfile.path, "#{basedir}/templates/default/#{::File.basename(f)}"
+            end            
+          end
+        end
+      end
+      
+      def json file=nil, &block
+        if @json_file
+          @json_file
+        else
+          if file
+            if ::File.file? file
+              ::File.cp file, "#{Default.tmp_path}/dr_configure/dna.json"
+            elsif file.is_a?(String)
+              ::File.open("#{Default.tmp_path}/dr_configure/dna.json", "w+"){|tf| tf << file } # is really a string
+            else
+              raise <<-EOM
+                Your json must either point to a file that exists or a string. Please check your configuration and try again
+              EOM
+            end
+            @json_file = "#{Default.tmp_path}/dr_configure/dna.json"
+          else
+            unless @recipe
+              @recipe = ChefRecipe.new
+              @recipe.instance_eval &block if block
+              @recipe.recipes(recipe_files.empty? ? ["poolparty"] : ["poolparty", "main"])
+              ::File.open("#{Default.tmp_path}/dr_configure/dna.json", "w+") {|f| f << @recipe.options.to_json }
+              @json_file = "#{Default.tmp_path}/dr_configure/dna.json"
+            end
+          end
+        end
+      end
+      
+      def include_recipes *recps
+        unless recps.empty?
+          recps.each do |rcp|
+            Dir[::File.expand_path(rcp)].each do |f|              
+              added_recipes << f
+            end            
+          end
+        end
+      end
+      
+      def config file=""
+        if @config_file
+          @config_file
+        else
+          if ::File.file? file
+            @config_file = file
+          else
+            conf_string = if file.empty?
+# default config
+            <<-EOE
+cookbook_path     "/etc/chef/cookbooks"
+node_path         "/etc/chef/nodes"
+log_level         :info
+file_store_path  "/etc/chef"
+file_cache_path  "/etc/chef"
+            EOE
+            else
+              open(file).read
+            end
+            ::File.open("#{Default.tmp_path}/dr_configure/chef_config.rb", "w+") do |tf|
+              tf << conf_string
+            end
+            @config_file = "#{Default.tmp_path}/dr_configure/chef_config.rb"
+          end
+        end
+      end
+      
+      def added_recipes
+        @added_recipes ||= []
+      end
+      
+      def after_create
+        before_configure
+      end
+      
+      def before_configure
+        config
+        json
+        
+        if ::File.directory?("/etc/chef")
+          ::Suitcase::Zipper.add("/etc/chef/cookbooks/*", "chef/recipes")
+          ::Suitcase::Zipper.add("/etc/chef/dna.json", "chef/json")
+          ::Suitcase::Zipper.add("/etc/chef/solo.rb", "chef/")
+        end
+        
+        ::Suitcase::Zipper.add(@config_file, "chef")
+        added_recipes.each do |rcp|
+          # ::FileUtils.cp_r rcp, "/tmp/poolparty/dr_configure/recipes/"
+          ::Suitcase::Zipper.add(rcp, "chef/recipes")
+        end
+        
+        ::Suitcase::Zipper.add(@json_file, "chef/json")
+        configure_commands ["cp -f /var/poolparty/dr_configure/chef/json/dna.json /etc/chef/dna.json"]
+
+        recipe_files.each do |rf|
+          ::FileUtils.mkdir_p "/tmp/poolparty/dr_configure/recipes/#{::File.basename(rf)}"
+          ::FileUtils.cp_r rf, "/tmp/poolparty/dr_configure/recipes/#{::File.basename(rf)}"
+          # ::Suitcase::Zipper.add(rf, "chef/recipes") 
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/deploydirectory.rb

@@ -5,64 +5,40 @@
   the master instance of the cloud. This enables you to send a directory
   up to the cloud and let the master host it for the remote slaves
 =end
-module PoolParty    
+module PoolParty
   class Deploydirectory
-        
-    virtual_resource(:deploydirectory) do
+    
+    virtual_resource(:deploy_directory) do
       
-      def loaded(opts={}, parent=self)
-        package_directory        
-        unpack_directory
-        sync_directories
-      end
-            
-      def package_directory
-        path = ::File.join( Base.tmp_path, "#{::File.basename(from_dir)}.tar.gz" )
-        archive_name = "#{::File.basename(name).dir_safe}.tar.gz"
-        cmd = "cd #{::File.expand_path(from_dir)} && tar -czf #{archive_name} . && mv #{archive_name} #{Base.tmp_path}"
-        Kernel.system(cmd) unless testing
+      def loaded(opts={}, &block)        
+        # raise(Exception.new("You must include a directory for the git repos set by :at")) if at?.nil?
+        # opts.has_key?(:at) ? at(opts.delete(:at)) : raise(Exception.new("You must include a directory for the git repos set by :at"))
+
+        package_deploy_directory
+        add_unpack_directory
       end
       
-      def unpack_directory
-        execute_on_master do
-          has_exec({:name => "deploy-directory-#{name}", :requires => get_directory("#{cwd}"), :cwd => cwd}) do
-            #  && rm #{Base.tmp_path}/#{parent.name.dir_safe}.tar.gz
-            archive_name = "#{::File.basename(name).dir_safe}.tar.gz"
-            command "cd #{cwd}; tar -zxf #{Base.remote_storage_path}/#{archive_name}; rm #{Base.remote_storage_path}/#{archive_name}; chown #{owner} #{::File.basename(name).dir_safe}"
-            onlyif "test -f #{Base.remote_storage_path}/#{archive_name}"
-          end
-        end
+      def package_deploy_directory
+        ::Suitcase::Zipper.add(sync_dir, "user_directory/")
       end
       
-      def sync_directories
-        execute_on_node do
-          has_rsyncmirror(:dir => cwd, :name => "deploydirectory-#{name}")
+      def add_unpack_directory
+        has_directory("#{::File.dirname(basedir)}")
+        
+        has_exec("unpack-#{::File.basename(basedir)}-deploy-directory") do
+          requires get_directory("#{::File.dirname(basedir)}")
+          cwd basedir
+          onlyif "test -f #{basedir}/#{sync_dir}"
+          command "cd #{cwd}; cp -R /var/poolparty/dr_configure/user_directory/#{sync_dir}; rm -rf /var/poolparty/dr_configure/#{name}"
         end
       end
-      
+            
       def from(dir)
-        from_dir (dir.include?(" ") ? dir.gsub(/[ ]/, '') : dir)
+        sync_dir dir
       end
       
       def to(dir)
-        cwd dir
-        name dir
-        has_directory(:name => "#{dir}", 
-                      :requires => get_directory("#{::File.dirname(dir)}"), 
-                      :owner => owner,
-                      :mode => mode)
-      end
-      
-      # Since git is not a native type, we have to say which core resource
-      # it is using to be able to require it
-      def class_type_name
-        "exec"
-      end
-      
-      # Because we are requiring an exec, instead of a built-in package of the git, we have to overload
-      # the to_s method and prepend it with the same name as above
-      def key
-        "deploy-directory-#{name}"
+        basedir dir
       end
       
     end

data/lib/poolparty/plugins/git.rb

@@ -9,26 +9,41 @@ module PoolParty
     
     virtual_resource(:git_repos) do
       
-      def loaded(opts={}, parent=self, &block)
-        has_git
-        has_git_repos
-      end
-            
-      def has_git_repos        
-        has_exec({:name => key, :requires => [get_directory("#{working_dir}"), get_package("git-core")] }) do
-          command requires_user ? "git clone #{requires_user}@#{source} #{working_dir}" : "cd #{working_dir} && git clone #{source}"
+      def loaded(opts={}, &block)        
+        raise(Exception.new("You must include a directory for the git repos set by :at")) if at?.nil?
+        # opts.has_key?(:at) ? at(opts.delete(:at)) : raise(Exception.new("You must include a directory for the git repos set by :at"))
+        # opts.has_key?(:source) ? git_repos(opts.delete(:source) || opts[:name]) : raise(Exception.new("You must include the git source set by :source"))
+        has_package("git-core")
+        has_git_repository
+      end
+
+      def has_git_repository
+        
+        has_directory(::File.dirname(working_dir))
+        has_directory(:name => "#{working_dir}", :requires => get_directory("#{::File.dirname(working_dir)}"))
+        
+        has_exec(:name => "git-#{name}", :requires => [get_directory("#{working_dir}"), get_package("git-core")] ) do
+          # Cloud, GitRepos, Exec
+          command parent.requires_user? ? "git clone #{requires_user}@#{source} #{working_dir}" : "cd #{working_dir} && git clone #{source}"
           cwd "#{working_dir if working_dir}"
           creates creates_dir
         end
-        has_exec(:name => "update-#{name}") do
-          cwd ::File.dirname( creates_dir )
+        has_exec(:name => "update-#{name}", :cwd => ::File.dirname( creates_dir )) do          
           command "git pull"
-        end                
+        end
+        if owner?
+          has_exec(:name => "chown-#{name}", :cwd => ::File.dirname( creates_dir )) do
+            command "chown #{owner} * -R"
+          end
+        end
+      end
+      
+      def git_repos(src)
+        source src
       end
       
       def at(dir)
         working_dir dir
-        has_directory(:name => "#{dir}", :requires => get_directory("#{::File.dirname(dir)}"))
       end
       
       def to(dir)
@@ -45,12 +60,6 @@ module PoolParty
         "exec"
       end
       
-      # Because we are requiring an exec, instead of a built-in package of the git, we have to overload
-      # the to_s method and prepend it with the same name as above
-      def key
-        "git-#{name}"
-      end
-      
     end
     
   end

data/lib/poolparty/plugins/line_in_file.rb

@@ -0,0 +1,13 @@
+module PoolParty    
+  class LineInFile
+    
+    virtual_resource(:line_in_file) do
+      def loaded(opts={}, &block)
+        has_exec "line_in_#{file}" do
+          command "grep -q \'#{line.safe_quote}\' #{file} || echo \'#{line.safe_quote}\' >> #{file}"
+        end
+      end
+    end
+    
+  end
+end

data/lib/poolparty/plugins/rsyncmirror.rb

@@ -3,11 +3,10 @@ module PoolParty
         
     virtual_resource(:rsyncmirror) do
       
-      def loaded(opts={}, parent=self)
-        @parent = parent        
+      def loaded(opts={})
         execute_on_node do
           @dir = dir || name
-          has_exec(opts.merge({:command => "#{cloud.remote_rsync_command} --no-implied-dirs --delete-excluded #{Base.user}@master:#{@dir}/ #{@dir}/".safe_quote, :name => "rsync-#{name}"}))
+          has_exec(opts.merge({:command => "#{cloud.remote_rsync_command} --no-implied-dirs --delete-excluded #{Default.user}@master:#{@dir}/ #{@dir}/".safe_quote, :name => "rsync-#{name}"}))
         end
       end