fairchild-poolparty 1.1.3

data/lib/poolparty/net/remoter_bases/ec2/ec2.rb

@@ -0,0 +1,235 @@
+=begin rdoc
+  EC2 Remoter Base
+  
+  This serves as the basis for running PoolParty on Amazon's ec2 cloud
+  cluster. 
+=end
+require "date"
+require "#{::File.dirname(__FILE__)}/ec2_response_object"
+
+begin
+  require 'EC2'
+rescue LoadError
+  puts <<-EOM
+Error: In order to use ec2, you need to install the amazon-ec2 gem
+
+Ec2 is the default remoter base for PoolParty. If you intend on using
+a different remoter base, specify it with:
+
+using :remoter_name
+
+in your config file, otherwise, to continue install grempe-amazon-ec2 with
+
+sudo gem install grempe-amazon-ec2 --source http://gems.github.com
+EOM
+end
+  
+class String
+  def convert_from_ec2_to_ip
+    self.gsub(/.compute-1.amazonaws.com*/, '').gsub(/ec2-/, '').gsub(/-/, '.')
+  end
+  def parse_datetime
+    DateTime.parse( self.chomp ) rescue self
+  end
+end
+  
+module PoolParty    
+  module Remote
+    class Ec2 < Remote::RemoterBase
+
+      def self.launch_new_instance!(o = options)
+        raise "You must pass a keypair to launch an instance, or else you wont be able to login. options = #{o.inspect}" if !o[:keypair]
+        instance = ec2(o).run_instances(
+          :image_id => o[:ami],
+          :user_data => o[:user_data],
+          :minCount => 1,
+          :maxCount => o[:num],
+          :key_name => o[:keypair],
+          :availability_zone => o[:availabilty_zone],
+          :instance_type => o[:size],
+          :group_id => o[:security_group])
+
+        begin
+          h = EC2ResponseObject.get_hash_from_response(instance.instancesSet.item.first)
+          #h = instance.instancesSet.item.first
+        rescue Exception => e
+          h = EC2ResponseObject.get_hash_from_response(instance) rescue instance
+          # h = instance
+        end
+        h
+      end
+      # Terminate an instance by id
+      def self.terminate_instance!(o={})  #NOTE: maybe we should not allow this command wihtout an instance_idˇ
+        ec2(o).terminate_instances(:instance_id => o[:instance_id])
+      end
+      # Describe an instance's status
+      def self.describe_instance(o={})
+        return describe_instances.first if o[:instance_id].nil?
+        describe_instances.detect {|a| a[:name] == o[:instance_id] || a[:ip] == o[:instance_id] || a[:instance_id] == o[:instance_id] }
+      end
+      def self.describe_instances(o={})
+        id = 0
+        get_instances_description(o).each_with_index do |h,i|
+          if h[:status] == "running"
+            inst_name = id == 0 ? "master" : "node#{id}"
+            id += 1
+          else
+            inst_name = "#{h[:status]}_node#{i}"
+          end
+          h.merge!({
+            :name => inst_name,
+            :hostname => h[:ip],
+            :ip => h[:ip].convert_from_ec2_to_ip,
+            :index => i,  #TODO MF get the instance id from the aws result instead
+            :launching_time => (h[:launching_time])
+          })
+        end.sort {|a,b| a[:index] <=> b[:index] }
+      end
+      
+      def self.ec2(o={})
+        @ec2 ||= EC2::Base.new( :access_key_id => o[:access_key], 
+                                :secret_access_key => o[:secret_access_key]
+                              )
+      end
+      # Get the ec2 description for the response in a hash format
+      def self.get_instances_description(o={})
+        EC2ResponseObject.get_descriptions(ec2(o).describe_instances)
+      end
+      def get_descriptions(o={})
+        self.class.get_descriptions(o)
+      end
+      
+      # Class method helpers
+      def self.aws_keys
+        unless @access_key && @secret_access_key          
+          aws_keys = {}
+          aws_keys = YAML::load( File.open('/etc/poolparty/aws_keys.yml') ) rescue 'No aws_keys.yml file.   Will try to use enviornment variables'
+          @access_key ||= aws_keys[:access_key] || ENV['AMAZON_ACCESS_KEY_ID'] || ENV['AWS_ACCESS_KEY']
+          @secret_access_key ||= aws_keys[:secret_access_key] || ENV['AMAZON_SECRET_ACCESS_KEY'] || ENV['AWS_SECRET_ACCESS_KEY']
+        end
+        [@access_key, @secret_access_key]
+      end
+
+      def after_launch_master(inst=nil)
+        instance = master
+        vputs "Running tasks after launching the master"
+        begin
+          # when_no_pending_instances do
+            if instance
+              attach_volume(instance)
+              # Let's associate the address LAST so that we can still connect to the instance
+              # for the other tasks here
+              associate_address(instance)
+              reset_remoter_base!
+            end
+          # end
+        rescue Exception => e        
+          vputs "Error in after_launch_master: #{e}"
+        end
+        reset_remoter_base!
+        when_all_assigned_ips {wait "5.seconds"}
+      end
+    
+      # Attach a volume to the instance
+      # DEPRECATE this relies on master.  master will be removed in next major release.  This method will be in ec2_remote_instance instead, or require an instance id
+      def attach_volume(instance=nil)
+        if ebs_volume_id
+          vputs "Attaching volume #{ebs_volume_id} to the master at #{ebs_volume_device}"
+          instance = master        
+          ec2.attach_volume(:volume_id => ebs_volume_id, :instance_id => instance.instance_id, :device => ebs_volume_device) if ebs_volume_id && ebs_volume_mount_point
+        end
+      end
+      # Associate an address with the instance using ec2
+      # DEPRECATE relies on master
+      def associate_address(instance=nil)
+        if set_master_ip_to
+          dputs "Associating master with #{set_master_ip_to}"
+          instance = master
+          ec2.associate_address(:instance_id => instance.instance_id, :public_ip => set_master_ip_to) if set_master_ip_to
+        end
+      end
+
+      # Help create a keypair for the cloud
+      # This is a helper to create the keypair and add them to the cloud for you
+      def create_keypair
+        return false unless keypair
+        unless ::File.exists?( new_keypair_path )
+          FileUtils.mkdir_p ::File.dirname( new_keypair_path )
+          vputs "Creating keypair: #{keypair} in #{new_keypair_path}"
+          Kernel.system "ec2-add-keypair #{keypair} > #{new_keypair_path} && chmod 600 #{new_keypair_path}"
+        end
+      end
+    
+      # wrapper for remote base to perform a snapshot backup for the ebs volume
+      def create_snapshot
+        return nil if ebs_volume_id.nil?
+        ec2.create_snapshot(:volume_id => ebs_volume_id)
+      end
+    
+      # EC2 connections
+      def ec2(o={})
+        @ec2 ||= self.class.ec2(o)
+      end
+    
+      # These are tasks that run before the configuration runs
+      def before_configuration_tasks
+        if set_master_ip_to && master.ip && master.ip.to_s != set_master_ip_to.to_s
+          associate_address(master)
+          reset_remoter_base!
+      
+          when_no_pending_instances do
+            when_all_assigned_ips do
+              vputs "Associated master with #{set_master_ip_to}"
+            end
+          end
+        end
+      
+      end
+      def has_cert_and_key?
+        pub_key && private_key
+      end
+      # The keys are used only for puppet certificates
+      # and are only used for EC2.
+      # Public key 
+      def pub_key
+        @pub_key ||= ENV["EC2_CERT"] ? ENV["EC2_CERT"] : nil
+      end
+      # Private key
+      def private_key
+        @private_key ||= ENV["EC2_PRIVATE_KEY"] ? ENV["EC2_PRIVATE_KEY"] : nil
+      end
+    
+      def custom_minimum_runnable_options
+        [:ami, :availabilty_zone, :security_group]
+      end
+
+      # Hook
+      #TODO#: Change this so they match with the cap tasks
+      def custom_install_tasks_for(o)        
+        [
+          # "if [ -z $(grep -v '#' /etc/hosts | grep '#{o.name}') ]; then echo \"$(curl http://169.254.169.254/latest/meta-data/public-ipv4) #{o.name}\" >> /etc/hosts; fi",
+          "if [ -z \"$(grep -v '#' /etc/hosts | grep '#{o.name}')\" ]; then echo '127.0.0.1 #{o.name}' >> /etc/hosts; fi",
+          "hostname #{o.name}",
+          "echo #{o.name} > /etc/hostname"
+        ]
+      end
+    
+      def after_install_tasks_for(o)
+        [
+          # "cd /var/poolparty && wget http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem -O amazon-ec2.gem 2>&1",
+          # "/usr/bin/gem install --no-ri --no-rdoc amazon-ec2.gem 2>&1"
+        ]
+      end
+
+      def custom_configure_tasks_for(o)
+        [
+        ]
+      end
+
+      def reset_base!
+        @describe_instances = @cached_descriptions = nil
+      end      
+    end
+        
+  end
+end

data/lib/poolparty/net/remoter_bases/ec2/ec2_remote_instance.rb

@@ -0,0 +1,53 @@
+module PoolParty  
+  module Remote
+    class Ec2RemoteInstance < RemoteInstance
+      include Dslify
+      
+      attr_reader :my_cloud, :uniquely_identifiable_by, :found_at
+
+      
+      # A new instance will be created from the passed in hash.  
+      # This hash of passed in values will be converted to methods on this instance.
+      # The parent clouds describe_instances list will be searched for the first one matching any of this instance's provided unique identifiers.
+      # If an instance is found, this instance's properties will be set to the properties provided
+      # If the found instance has properties of the same key as the provided options, the found instance's values will override the passed in options
+      def initialize(opts={}, prnt=Ec2.new)
+        @uniquely_identifiable_by = [:ip, :name, :dns_name, :instance_id]
+        @original_options = opts
+        @my_cloud = prnt
+        super(opts, prnt)
+        find_myself(@uniquely_identifiable_by && opts.keys) if prnt.respond_to?(:describe_instances)
+      end
+   
+      # Search the clouds describe_instances list for the first match on one of this nodes unique identifiers
+      def find_myself(unique_identifiers = @original_options.keys)
+        description_hash = @my_cloud.describe_instances.detect do|node|
+          unique_identifiers.detect{|identifier_key| node[identifier_key] == options[identifier_key]
+            }
+        end
+        return nil if description_hash.blank?
+        @found_at = Time.now
+        self.set_vars_from_options(description_hash)
+        self
+      end
+      
+      # Is this instance running?
+      def running?
+        !(status =~ /running/).nil?
+      end
+      # Is this instance pending?
+      def pending?
+        !(status =~ /pending/).nil?
+      end
+      # Is this instance terminating?
+      def terminating?
+        !(status =~ /shutting/).nil?
+      end
+      # Has this instance been terminated?
+      def terminated?
+        !(status =~ /terminated/).nil?
+      end
+      
+    end
+  end
+end

data/lib/poolparty/net/remoter_bases/ec2/ec2_response_object.rb

@@ -0,0 +1,64 @@
+# Convenience class to convert standard amazon-ec2 responses from their camel cased style to a hash using underscore style.
+# For example: instanceId to instance_id
+class EC2ResponseObject
+  def self.get_descriptions(resp)      
+    rs = get_instance_from_response(resp)
+    group = get_group_from_response(resp)
+    
+    # puts rs.methods.sort - rs.ancestors.methods
+    out = begin
+      if rs.respond_to?(:instancesSet)
+        [EC2ResponseObject.get_hash_from_response(rs.instancesSet.item, group)]
+      else
+        rs.collect {|r| 
+          if r.instancesSet.item.class == Array
+            r.instancesSet.item.map {|t| EC2ResponseObject.get_hash_from_response(t, group)}
+          else
+            [EC2ResponseObject.get_hash_from_response(r.instancesSet.item, group)]
+          end            
+        }.flatten.reject {|a| a.nil? }
+      end
+    rescue Exception => e
+      # Really weird bug with amazon's ec2 gem
+      rs.collect {|r| EC2ResponseObject.get_hash_from_response(r)}.reject {|a| a.nil? } rescue []
+    end
+
+    out
+  end
+  def self.get_instance_from_response(resp)
+    begin
+      rs = resp.reservationSet.item unless resp.reservationSet.nil?
+      rs ||= resp.DescribeInstancesResponse.reservationSet.item
+      rs ||= rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      rs.reject! {|a| a.nil? || a.empty? }
+    rescue Exception => e
+      resp
+    end
+    rs
+  end
+  def self.get_group_from_response(resp)
+    begin
+      resp = resp.reservationSet.item.first if resp.reservationSet.item.is_a?(Array)
+      group = resp.reservationSet.item.groupSet.item.groupId unless resp.reservationSet.nil?
+      group ||= resp.groupSet.item[0].groupId rescue nil
+      group ||= resp.DescribeInstancesResponse.reservationSet.item.groupSet.item.groupId
+      #rs ||= rs.respond_to?(:instancesSet) ? rs.instancesSet : rs
+      #rs.reject! {|a| a.nil? || a.empty? }
+    rescue Exception => e
+      resp
+    end
+    group
+  end
+  def self.get_hash_from_response(resp, group = 'default')
+      {
+        :instance_id => resp.instanceId,
+        :name => resp.instanceId, 
+        :ip => resp.dnsName || "not-assigned",
+        :status => resp.instanceState.name,
+        :launching_time => resp.launchTime.parse_datetime,
+        :internal_ip => resp.privateDnsName,
+        :keypair => resp.keyName,
+        :security_group => group
+      }        
+  end
+end

data/lib/poolparty/plugins/chef.rb

@@ -0,0 +1,168 @@
+require "tempfile"
+module PoolParty
+  class ChefRecipe
+    include Dslify
+  end
+  class Chef
+    
+    plugin :chef do
+      def before_load(o, &block)        
+        bootstrap_gems "chef", "ohai"
+        bootstrap_commands [
+          "mkdir -p /etc/chef/cookbooks /etc/chef/cache"
+        ]
+      end
+      
+      def loaded o={}, &block
+      end
+      
+      def recipe_files
+        @recipe_files ||= []
+      end
+      
+      def basedir
+        @basedir ||= "#{Default.tmp_path}/dr_configure/chef/recipes/main"
+      end
+      
+      def recipe file=nil, o={}, &block        
+        if file
+          ::FileUtils.mkdir_p "#{basedir}/recipes" unless ::File.directory? basedir
+          
+          unless ::File.file?(file)
+            tfile = Tempfile.new("main-poolparty-recipe")
+            tfile << file # copy the string into the temp file
+            file = tfile.path
+          end
+          
+          ::File.cp file, "#{basedir}/recipes/default.rb"
+          
+          templates o[:templates] if o[:templates]
+          
+          recipe_files << basedir
+        # TODO: Enable neat syntax from within poolparty
+        else
+          raise <<-EOR
+            PoolParty currently only supports passing recipes as files. Please specify a file in your chef block and try again"
+          EOR
+        end
+      end
+      
+      def templates templates=[]
+        if templates
+          ::FileUtils.mkdir_p "#{basedir}/templates/default/"
+          templates.each do |f|
+            if ::File.file?(f)
+              ::File.cp f, "#{basedir}/templates/default/#{::File.basename(f)}"
+            elsif ::File.directory?(f)
+              Dir["#{f}/*"].each {|f| ::File.cp f, "#{basedir}/templates/default/#{::File.basename(f)}" }
+            else
+              tfile = Tempfile.new("main-poolparty-recipe")
+              tfile << f # copy the string into the temp file
+              ::File.cp tfile.path, "#{basedir}/templates/default/#{::File.basename(f)}"
+            end            
+          end
+        end
+      end
+      
+      def json file=nil, &block
+        if @json_file
+          @json_file
+        else
+          if file
+            if ::File.file? file
+              ::File.cp file, "#{Default.tmp_path}/dr_configure/dna.json"
+            elsif file.is_a?(String)
+              ::File.open("#{Default.tmp_path}/dr_configure/dna.json", "w+"){|tf| tf << file } # is really a string
+            else
+              raise <<-EOM
+                Your json must either point to a file that exists or a string. Please check your configuration and try again
+              EOM
+            end
+            @json_file = "#{Default.tmp_path}/dr_configure/dna.json"
+          else
+            unless @recipe
+              @recipe = ChefRecipe.new
+              @recipe.instance_eval &block if block
+              @recipe.recipes(recipe_files.empty? ? ["poolparty"] : ["poolparty", "main"])
+              ::File.open("#{Default.tmp_path}/dr_configure/dna.json", "w+") {|f| f << @recipe.options.to_json }
+              @json_file = "#{Default.tmp_path}/dr_configure/dna.json"
+            end
+          end
+        end
+      end
+      
+      def include_recipes *recps
+        unless recps.empty?
+          recps.each do |rcp|
+            Dir[::File.expand_path(rcp)].each do |f|              
+              added_recipes << f
+            end            
+          end
+        end
+      end
+      
+      def config file=""
+        if @config_file
+          @config_file
+        else
+          if ::File.file? file
+            @config_file = file
+          else
+            conf_string = if file.empty?
+# default config
+            <<-EOE
+cookbook_path     "/etc/chef/cookbooks"
+node_path         "/etc/chef/nodes"
+log_level         :info
+file_store_path  "/etc/chef"
+file_cache_path  "/etc/chef"
+            EOE
+            else
+              open(file).read
+            end
+            ::File.open("#{Default.tmp_path}/dr_configure/chef_config.rb", "w+") do |tf|
+              tf << conf_string
+            end
+            @config_file = "#{Default.tmp_path}/dr_configure/chef_config.rb"
+          end
+        end
+      end
+      
+      def added_recipes
+        @added_recipes ||= []
+      end
+      
+      def after_create
+        before_configure
+      end
+      
+      def before_configure
+        config
+        json
+        
+        if ::File.directory?("/etc/chef")
+          ::Suitcase::Zipper.add("/etc/chef/cookbooks/*", "chef/recipes")
+          ::Suitcase::Zipper.add("/etc/chef/dna.json", "chef/json")
+          ::Suitcase::Zipper.add("/etc/chef/solo.rb", "chef/")
+        end
+        
+        ::Suitcase::Zipper.add(@config_file, "chef")
+        added_recipes.each do |rcp|
+          # ::FileUtils.cp_r rcp, "/tmp/poolparty/dr_configure/recipes/"
+          ::Suitcase::Zipper.add(rcp, "chef/recipes")
+        end
+        
+        ::Suitcase::Zipper.add(@json_file, "chef/json")
+        configure_commands ["cp -f /var/poolparty/dr_configure/chef/json/dna.json /etc/chef/dna.json"]
+
+        recipe_files.each do |rf|
+          ::FileUtils.mkdir_p "/tmp/poolparty/dr_configure/recipes/#{::File.basename(rf)}"
+          ::FileUtils.cp_r rf, "/tmp/poolparty/dr_configure/recipes/#{::File.basename(rf)}"
+          # ::Suitcase::Zipper.add(rf, "chef/recipes") 
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/deploydirectory.rb

@@ -0,0 +1,47 @@
+=begin rdoc
+  DeployDirectory
+  
+  Deploy directory will tar.gz a local directory and sync it up to 
+  the master instance of the cloud. This enables you to send a directory
+  up to the cloud and let the master host it for the remote slaves
+=end
+module PoolParty
+  class Deploydirectory
+    
+    virtual_resource(:deploy_directory) do
+      
+      def loaded(opts={}, &block)        
+        # raise(Exception.new("You must include a directory for the git repos set by :at")) if at?.nil?
+        # opts.has_key?(:at) ? at(opts.delete(:at)) : raise(Exception.new("You must include a directory for the git repos set by :at"))
+
+        package_deploy_directory
+        add_unpack_directory
+      end
+      
+      def package_deploy_directory
+        ::Suitcase::Zipper.add(sync_dir, "user_directory/")
+      end
+      
+      def add_unpack_directory
+        has_directory("#{::File.dirname(basedir)}")
+        
+        has_exec("unpack-#{::File.basename(basedir)}-deploy-directory") do
+          requires get_directory("#{::File.dirname(basedir)}")
+          cwd basedir
+          onlyif "test -f #{basedir}/#{sync_dir}"
+          command "cd #{cwd}; cp -R /var/poolparty/dr_configure/user_directory/#{sync_dir}; rm -rf /var/poolparty/dr_configure/#{name}"
+        end
+      end
+            
+      def from(dir)
+        sync_dir dir
+      end
+      
+      def to(dir)
+        basedir dir
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/git.rb

@@ -0,0 +1,66 @@
+module PoolParty    
+  class GitResource
+    
+    virtual_resource(:git) do
+      def loaded(*args)
+        has_package(:name => "git-core")
+      end
+    end
+    
+    virtual_resource(:git_repos) do
+      
+      def loaded(opts={}, &block)        
+        raise(Exception.new("You must include a directory for the git repos set by :at")) if at?.nil?
+        # opts.has_key?(:at) ? at(opts.delete(:at)) : raise(Exception.new("You must include a directory for the git repos set by :at"))
+        # opts.has_key?(:source) ? git_repos(opts.delete(:source) || opts[:name]) : raise(Exception.new("You must include the git source set by :source"))
+        has_package("git-core")
+        has_git_repository
+      end
+
+      def has_git_repository
+        
+        has_directory(::File.dirname(working_dir))
+        has_directory(:name => "#{working_dir}", :requires => get_directory("#{::File.dirname(working_dir)}"))
+        
+        has_exec(:name => "git-#{name}", :requires => [get_directory("#{working_dir}"), get_package("git-core")] ) do
+          # Cloud, GitRepos, Exec
+          command parent.requires_user? ? "git clone #{requires_user}@#{source} #{working_dir}" : "cd #{working_dir} && git clone #{source}"
+          cwd "#{working_dir if working_dir}"
+          creates creates_dir
+        end
+        has_exec(:name => "update-#{name}", :cwd => ::File.dirname( creates_dir )) do          
+          command "git pull"
+        end
+        if owner?
+          has_exec(:name => "chown-#{name}", :cwd => ::File.dirname( creates_dir )) do
+            command "chown #{owner} * -R"
+          end
+        end
+      end
+      
+      def git_repos(src)
+        source src
+      end
+      
+      def at(dir)
+        working_dir dir
+      end
+      
+      def to(dir)
+        at(dir)
+      end
+      
+      def creates_dir
+        "#{::File.join( working_dir, ::File.basename(source, ::File.extname(source)) )}/.git"
+      end
+      
+      # Since git is not a native type, we have to say which core resource
+      # it is using to be able to require it
+      def class_type_name
+        "exec"
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/line_in_file.rb

@@ -0,0 +1,13 @@
+module PoolParty    
+  class LineInFile
+    
+    virtual_resource(:line_in_file) do
+      def loaded(opts={}, &block)
+        has_exec "line_in_#{file}" do
+          command "grep -q \'#{line.safe_quote}\' #{file} || echo \'#{line.safe_quote}\' >> #{file}"
+        end
+      end
+    end
+    
+  end
+end

data/lib/poolparty/plugins/rsyncmirror.rb

@@ -0,0 +1,28 @@
+module PoolParty    
+  class Rsyncmirror
+        
+    virtual_resource(:rsyncmirror) do
+      
+      def loaded(opts={})
+        execute_on_node do
+          @dir = dir || name
+          has_exec(opts.merge({:command => "#{cloud.remote_rsync_command} --no-implied-dirs --delete-excluded #{Default.user}@master:#{@dir}/ #{@dir}/".safe_quote, :name => "rsync-#{name}"}))
+        end
+      end
+            
+      # Since git is not a native type, we have to say which core resource
+      # it is using to be able to require it
+      def class_type_name
+        "exec"
+      end
+      
+      # Because we are requiring an exec, instead of a built-in package of the git, we have to overload
+      # the to_s method and prepend it with the same name as above
+      def key
+        "rsync-#{name}"
+      end
+      
+    end
+    
+  end
+end