poolparty 0.0.4 → 0.2.6

data/lib/poolparty/net/remoter_base.rb

@@ -0,0 +1,117 @@
+module PoolParty
+
+  def register_remote_base(*args)
+    args.each do |arg|
+      (remote_bases << "#{arg}".downcase.to_sym)
+    end
+  end
+  
+  def remote_bases
+    $remote_bases ||= []
+  end  
+
+  module Remote    
+    # This class is the base class for all remote types
+    # Everything remoting-wise is derived from this class
+    module RemoterBaseMethods
+      # Required methods
+      # The next methods are required on all RemoteInstance types
+      # If your RemoteInstance type does not overwrite the following methods
+      # An exception will be raised and poolparty will explode into tiny little 
+      # pieces. Don't forget to overwrite these methods
+      # Launch a new instance
+      def launch_new_instance!
+        raise RemoteException.new(:method_not_defined, "launch_new_instance!")
+      end
+      # Terminate an instance by id
+      def terminate_instance!(id=nil)
+        raise RemoteException.new(:method_not_defined, "terminate_instance!")
+      end
+      # Describe an instance's status
+      def describe_instance(id=nil)
+        raise RemoteException.new(:method_not_defined, "describe_instance")
+      end
+      # Get instances
+      # The instances must have a status associated with them on the hash
+      def describe_instances
+        raise RemoteException.new(:method_not_defined, "describe_instances")
+      end
+      
+    end
+    module RemoterBase
+      # The following methods are inherent on the RemoterBase
+      # If you need to overwrite these methods, do so with caution
+      # Listing methods
+      def list_of_running_instances(list = list_of_nonterminated_instances)
+        list.select {|a| a.running? }
+      end
+      # Get a list of the pending instances
+      def list_of_pending_instances(list = list_of_nonterminated_instances)
+        list.select {|a| a.pending? }
+      end
+      # list of shutting down instances
+      def list_of_terminating_instances(list = remote_instances_list)
+        list.reject {|i| true if !i.terminating? }
+      end
+      # Get the instances that are non-master instances
+      def nonmaster_nonterminated_instances(list = list_of_nonterminated_instances)
+        list_of_nonterminated_instances.reject {|i| i.master? }
+      end
+      # list all the nonterminated instances
+      def list_of_nonterminated_instances(list = remote_instances_list)
+        list.reject {|i| i.terminating? || i.terminated? }
+      end
+      # Get instance by number
+      def get_instance_by_number(i=0, list = remote_instances_list)
+        name = (i.zero? ? "master" : "node#{i}")
+        list.select {|i| i.name == name }.first
+      end
+      def remote_instances_list
+        list_of_instances.map {|i| PoolParty::Remote::RemoteInstance.new(i, self) }
+      end
+      # List the instances for the current key pair, regardless of their states
+      # If no keypair is passed, select them all
+      def list_of_instances(keyp=nil)
+        key = keyp ? keyp : keypair
+        describe_instances.select {|a| key ? a[:keypair] == key : true } if describe_instances
+      end
+      # Instances
+      # Get the master from the cloud
+      def master
+        # remote_instances_list.select {|a| a.master }.first
+        @list = list_from_remote
+        @list.reject {|a| a unless a.name =~ /master/ }.first if @list.class != String
+      end
+      # Helpers
+      def create_keypair
+      end
+      # Reset the cache of descriptions
+      def reset!
+      end
+      def self.included(other)
+        PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)
+      end
+      
+      # Callback after loaded
+      def loaded_remoter_base        
+      end
+            
+      # Custom installation tasks
+      # Allow the remoter bases to attach their own tasks on the 
+      # installation process
+      def custom_install_tasks_for(a=nil)
+        []
+      end
+      # Custom configure tasks
+      # Allows the remoter bases to attach their own
+      # custom configuration tasks to the configuration process
+      def custom_configure_tasks_for(a=nil)
+        []
+      end
+      
+    end
+    
+  end
+end
+
+Dir["#{File.dirname(__FILE__)}/remote_bases/*.rb"].each {|base| require base }

data/lib/poolparty/plugins/gem_package.rb

@@ -0,0 +1,17 @@
+module PoolParty    
+  class Gem
+        
+    virtual_resource(:gempackage) do
+      
+      def loaded(opts={})
+        has_exec(:name => "gem-package-#{name}", :cwd => "/tmp", :path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin") do
+          command "gem install -y --no-ri --no-rdoc #{"--version \"#{version}\"" if @parent.version} #{"--source #{@parent.source}" if @parent.source} #{@parent.name}"
+          ifnot "gem list --local #{@parent.name} | grep #{@parent.name} #{"| grep #{@parent.version}" if @parent.version}"
+          requires "Package['rubygems']"
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/git.rb

@@ -0,0 +1,41 @@
+module PoolParty    
+  class Git
+        
+    virtual_resource(:git) do
+      
+      def loaded(opts={})
+        install_git
+        has_git_repos
+      end
+      
+      def install_git
+        has_package(:name => "git-core")
+      end
+      
+      def has_git_repos
+        with_options(:requires => 'Package["git-core"]') do
+          has_directory(:name => "#{path}")
+
+          exec({:name => "git-#{name}"}) do
+            command @parent.user ? "git clone #{@parent.user}@#{@parent.source} #{@parent.path}" : "git clone #{@parent.source} #{@parent.path}"
+            cwd "#{::File.dirname(@parent.path) if @parent.path}"
+            creates ["#{@parent.path}", "#{@parent.path}/.git"]
+          end
+
+          exec(:name => "git-update-#{name}", :cwd => "#{path}") do
+            command "git pull"
+            requires "Exec['git-#{@parent.name}']"
+          end
+
+          if symlink
+            has_file(:name => "#{symlink}") do
+              ensures @parent.path
+            end
+          end
+        end
+      end
+      
+    end
+    
+  end
+end

data/lib/poolparty/plugins/line.rb

@@ -0,0 +1,76 @@
+module PoolParty    
+  class Line
+        
+    define_resource(:line_in_file) do
+      # Should refactor
+      def has_line_in_file(line="line_in_file", file="file", opts={})
+        call_function <<-EOE
+        line {
+          '#{file}_line' :
+            file => '#{file}',
+            line => '#{line}',
+            #{opts.flush_out("\t", ",")}
+            ensure => 'present'
+        }
+        EOE
+      end
+      
+      def append_if_no_such_line(line="line", file="", refreshonly='false')
+        call_function <<-EOE
+        append_if_no_such_line {
+          '#{file}' :
+            file => '#{file}',
+            line => '#{line}',
+            refreshonly => #{refreshonly}
+        }
+        EOE
+      end
+      
+      def delete_lines_from(file="", pattern=//)
+        call_function <<-EOE
+        delete_lines {
+          '#{file}' :
+            file => '#{file}',
+            pattern => '#{pattern}'
+        }
+        EOE
+      end
+      
+                    
+      custom_function <<-EOF
+      define line($file, $line, $ensure = 'present', $notify=[]) {
+        case $ensure {
+          default: { err ( "unknown ensure value ${ensure}" ) }
+          present: {
+            exec {
+              "/usr/bin/env echo '${line}' >> '${file}'": unless => "/usr/bin/env grep -qFx '${line}' '${file}'",
+               notify => $notify
+              }
+            }
+          absent: {
+            exec {
+              "/usr/bin/env sed -i '' -e '/^${line}\$/d' '${file}'": onlyif => "/usr/bin/env grep -qFx '${line}' '${file}'",
+              notify => $notify
+            }
+          }
+        }
+      }
+      define append_if_no_such_line($file, $line, $refreshonly = 'false') {
+         exec { "/bin/echo '$line' >> '$file'":
+            unless      => "/bin/grep -Fxqe '$line' '$file'",
+            path        => "/bin",
+            refreshonly => $refreshonly,
+         }
+      }
+
+      define delete_lines($file, $pattern) {
+         exec { "sed -i -r -e '/$pattern/d' $file":
+            path   => "/bin",
+            onlyif => "/bin/grep -E '$pattern' '$file'",
+         }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/plugins/svn.rb

@@ -0,0 +1,48 @@
+module PoolParty    
+  class Svn
+        
+    define_resource(:svn) do
+      
+      def has_svnpath(opts={})
+        call_function <<-EOE
+        svnserve { #{opts[:name]}:
+          source => "#{opts[:source]}",
+          path => "#{opts[:path]}",
+          user => "#{opts[:user] || false}",
+          password => "#{opts[:password] || ""}"
+        }
+        EOE
+      end
+      
+      custom_function <<-EOF
+      # Serve subversion-based code from a local location.  The job of this
+      # module is to check the data out from subversion and keep it up to
+      # date, especially useful for providing data to your Puppet server.
+      define svnserve($source, $path, $user = false, $password = false) {
+          file { $path:
+              ensure => directory,
+              owner => root,
+              group => root
+          }
+          $svncmd = $user ? {
+              false => "/usr/bin/svn co --non-interactive $source/$name .",
+              default => "/usr/bin/svn co --non-interactive --username $user --password '$password' $source/$name ."
+          }   
+          exec { "svnco-$name":
+              command => $svncmd,
+              cwd => $path,
+              require => File[$path],
+              creates => "$path/.svn"
+          }
+          exec { "svnupdate-$name":
+              command => "/usr/bin/svn update",
+              require => Exec["svnco-$name"],
+              onlyif => '/usr/bin/svn status -u --non-interactive | /bin/grep "\*"',
+              cwd => $path
+          }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/pool/base.rb

@@ -0,0 +1,75 @@
+=begin rdoc
+  Base
+  This handles user interaction, loading the parameters, etc.
+=end
+require "open-uri"
+require "ftools"
+module PoolParty
+  class Base
+    include Configurable
+    extend MethodMissingSugar
+    
+    default_options({
+      :environment => "production",
+      :user => "root", # This should change here
+      :base_keypair_path => "~/.ec2",
+      :tmp_path => "tmp",
+      :remote_storage_path => "/var/poolparty",
+      :storage_directory => File.join(Dir.pwd, "tmp"),
+      :fileserver_base => "puppet:///files",
+      :base_config_directory => "/etc/poolparty",
+      :template_directory => "tmp/templates",
+      :template_path => "/var/lib/puppet/templates",
+      :default_specfile_name => "pool.spec",
+      :port => "80",
+      :forwarding_port => "8080",
+      :proxy_mode => "http",
+      :pool_logger_location => File.join(Dir.pwd, "logs"),
+      # EC2 Options
+      :ami => "ami-1cd73375" 
+    })
+        
+    # Class methods
+    class << self
+      def options(h={})
+        @options ||= default_options.merge(h)
+      end
+      # Get the access_key
+      def access_key
+        ENV["AWS_ACCESS_KEY_ID"] ? ENV["AWS_ACCESS_KEY_ID"] : load_keys_from_file[:access_key]
+      end
+      def secret_access_key
+        ENV["AWS_SECRET_ACCESS_ID"] ? ENV["AWS_SECRET_ACCESS_ID"] : load_keys_from_file[:secret_access_key]
+      end
+      def load_keys_from_file
+        @keys ||= get_working_key_file_locations ? YAML::load( open(get_working_key_file_locations).read ) : {}
+      end
+      # Store the keys in a yaml format to give the master access
+      # So that the master has access to the files
+      def store_keys_in_file
+        unless access_key.nil? || secret_access_key.nil?
+          write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
+        end
+      end
+      def reset!
+        @keys = nil
+      end
+      # Get the instance first instance file that exists on the system from the expected places
+      # denoted in the local_instances_list_file_locations
+      def get_working_key_file_locations
+        key_file_locations.reject {|f| f unless ::File.file?(f) }.first
+      end
+      # Expected places for the instances.list to be located at on the machine
+      def key_file_locations
+        [
+          ".ppkeys",
+          "#{Base.base_config_directory}/.ppkeys",
+          "#{Base.storage_directory}/ppkeys",          
+          "~/.ppkeys",
+          "ppkeys"
+        ]
+      end
+      
+    end
+  end    
+end

data/lib/poolparty/pool/cloud.rb

@@ -0,0 +1,164 @@
+require File.dirname(__FILE__) + "/plugin_model"
+require File.dirname(__FILE__) + "/resource"
+
+module PoolParty    
+  module Cloud
+    def cloud(name=:main, &block)
+      clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, self, &block))
+    end
+
+    def clouds
+      $clouds ||= {}
+    end
+    
+    def with_cloud(cl, opts={}, &block)
+      cl.options.merge!(opts)
+      cl.instance_eval &block if block
+    end
+    
+    class Cloud
+      attr_reader :name, :templates
+      include PoolParty::PluginModel
+      include PoolParty::Resources      
+      include PrettyPrinter
+      include Configurable
+      include CloudResourcer
+      # extend CloudResourcer
+      # Net methods
+      include PoolParty::Remote::RemoterBase
+      include Remote
+      
+      default_options({
+        :minimum_instances => 2,
+        :maximum_instances => 4,
+        :access_key => Base.access_key,
+        :secret_access_key => Base.secret_access_key,
+        :ec2_dir => ENV["EC2_HOME"],
+        :keypair => (ENV["KEYPAIR_NAME"].nil? || ENV["KEYPAIR_NAME"].empty?) ? nil : ENV["KEYPAIR_NAME"],
+        :ami => 'ami-44bd592d',
+        :polling_time => "30.seconds"
+      })
+      
+      def initialize(name, parent, &block)
+        @name = name
+        # store_block(&block)
+        set_parent(parent) if parent
+        self.instance_eval &block if block
+        # this can be overridden in the spec, but ec2 is the default
+        self.using :ec2
+      end
+                        
+      # Keypairs
+      # If the parent (pool) doesn't have a keypair defined on it, then generate one based on the 
+      # pool_name and the cloud_name
+      def keypair(*args)
+        has_keypair? ? options[:keypair] : generate_keypair(*args)
+      end
+      def has_keypair?
+        options.has_key?(:keypair) && options[:keypair]
+      end
+      def generate_keypair(*args)
+        options[:keypair] = args.length > 0 ? args[0] : "#{@parent.respond_to?(:name) ? @parent.name : ""}_#{@name}"
+      end
+      
+      # Prepare to send the new configuration to the instances
+      # First, let's make sure that our base directory is made
+      # Then copy the templates that have no other reference in
+      # a spec file. Make sure the keys are stored in a file
+      # For the master to have access to them
+      # Then, send the saved containing cloud instances to give the 
+      # remote master access to the cloud options that are required
+      # for the master to run checks
+      def prepare_to_configuration        
+        # clear_base_directory
+        make_base_directory
+        copy_misc_templates
+        Base.store_keys_in_file
+        Script.save!
+        copy_ssh_key # not my favorite...
+      end
+      
+      def copy_ssh_key
+        copy_file_to_storage_directory(full_keypair_path)
+      end
+      
+      # Build the new poolparty manifest
+      # Wrapping all of these requirements into the one 
+      # poolparty class.
+      # 
+      # TODO: Consider the benefits of moving all the manifest
+      # classes to separate files and keeping the containing
+      # references in the include
+      def build_and_store_new_config_file
+        @manifest = build_manifest
+        config_file = ::File.join(Base.storage_directory, "poolparty.pp")
+        ::File.open(config_file, "w+") do |file|
+          file << "class poolparty {"
+          file << @manifest
+          file << "}"
+        end
+      end      
+      
+      def copy_misc_templates
+        ["fileserver.conf", "namespaceauth.conf"].each do |f|
+          copy_file_to_storage_directory(::File.join(::File.dirname(__FILE__), "..", "templates", f))
+        end
+      end
+      
+      # Configuration files
+      def build_manifest
+        unless @build_manifest
+          reset_resources!
+          add_poolparty_base_requirements
+
+          @build_manifest = returning Array.new do |str|
+
+            str << resources_string_from_resources(resources)
+
+            # Refactor this into the resources method
+            # TODO
+            services.each do |service|
+              @cp = classpackage_with_self(service)
+              str << @cp.to_string
+              str << @cp.include_string
+            end
+
+            str << "# Custom functions"
+            str << Resources::CustomResource.custom_functions_to_string
+          end.join("\n")          
+        end
+        @build_manifest
+      end
+      
+      # To allow the remote instances to do their job,
+      # they need a few options to run, these are the required options
+      # to be saved on the remote "master" machine
+      def minimum_runnable_options
+        [:keypair, :minimum_instances, :maximum_instances]
+      end
+      
+      # Add all the poolparty requirements here
+      # NOTE: These are written as plugins in the lib/poolparty/base_packages directory
+      # for examples. 
+      # Also note that there is no block associated. This is because we have written
+      # all that is necessary in a method called enable
+      # which is called when there is no block
+      def add_poolparty_base_requirements        
+        heartbeat
+        haproxy
+        ruby
+        poolparty
+      end
+            
+      # Add to the services pool for the manifest listing
+      def add_service(serv)
+        services << serv
+      end
+      # Container for the services
+      def services
+        @services ||= []
+      end
+            
+    end
+  end  
+end