auser-poolparty 0.1.2 → 0.2.2

data/lib/poolparty/net/remoter_base.rb

@@ -0,0 +1,117 @@
+module PoolParty
+
+  def register_remote_base(*args)
+    args.each do |arg|
+      (remote_bases << "#{arg}".downcase.to_sym)
+    end
+  end
+  
+  def remote_bases
+    $remote_bases ||= []
+  end  
+
+  module Remote    
+    # This class is the base class for all remote types
+    # Everything remoting-wise is derived from this class
+    module RemoterBaseMethods
+      # Required methods
+      # The next methods are required on all RemoteInstance types
+      # If your RemoteInstance type does not overwrite the following methods
+      # An exception will be raised and poolparty will explode into tiny little 
+      # pieces. Don't forget to overwrite these methods
+      # Launch a new instance
+      def launch_new_instance!
+        raise RemoteException.new(:method_not_defined, "launch_new_instance!")
+      end
+      # Terminate an instance by id
+      def terminate_instance!(id=nil)
+        raise RemoteException.new(:method_not_defined, "terminate_instance!")
+      end
+      # Describe an instance's status
+      def describe_instance(id=nil)
+        raise RemoteException.new(:method_not_defined, "describe_instance")
+      end
+      # Get instances
+      # The instances must have a status associated with them on the hash
+      def describe_instances
+        raise RemoteException.new(:method_not_defined, "describe_instances")
+      end
+      
+    end
+    module RemoterBase
+      # The following methods are inherent on the RemoterBase
+      # If you need to overwrite these methods, do so with caution
+      # Listing methods
+      def list_of_running_instances(list = list_of_nonterminated_instances)
+        list.select {|a| a.running? }
+      end
+      # Get a list of the pending instances
+      def list_of_pending_instances(list = list_of_nonterminated_instances)
+        list.select {|a| a.pending? }
+      end
+      # list of shutting down instances
+      def list_of_terminating_instances(list = remote_instances_list)
+        list.reject {|i| true if !i.terminating? }
+      end
+      # Get the instances that are non-master instances
+      def nonmaster_nonterminated_instances(list = remote_instances_list)
+        list_of_nonterminated_instances.reject {|i| i.master? }
+      end
+      # list all the nonterminated instances
+      def list_of_nonterminated_instances(list = remote_instances_list)
+        list.reject {|i| i.terminating? || i.terminated? }
+      end
+      # Get instance by number
+      def get_instance_by_number(i=0, list = remote_instances_list)
+        name = (i.zero? ? "master" : "node#{i}")
+        list.select {|i| i.name == name }.first
+      end
+      def remote_instances_list
+        list_of_instances.map {|i| PoolParty::Remote::RemoteInstance.new(i, self) }
+      end
+      # List the instances for the current key pair, regardless of their states
+      # If no keypair is passed, select them all
+      def list_of_instances(keyp=nil)
+        key = keyp ? keyp : keypair
+        describe_instances.select {|a| key ? a[:keypair] == key : true }
+      end
+      # Instances
+      # Get the master from the cloud
+      def master
+        # remote_instances_list.select {|a| a.master }.first
+        @list = list_from_remote
+        @list.reject {|a| a unless a.name =~ /master/ }.first if @list.class != String
+      end
+      # Helpers
+      def create_keypair
+      end
+      # Reset the cache of descriptions
+      def reset!
+      end      
+      def self.included(other)
+        PoolParty.register_remote_base(self.class.to_s.downcase.to_sym)
+      end
+      
+      # Callback after loaded
+      def loaded_remoter_base        
+      end
+            
+      # Custom installation tasks
+      # Allow the remoter bases to attach their own tasks on the 
+      # installation process
+      def custom_install_tasks_for(a=nil)
+        []
+      end
+      # Custom configure tasks
+      # Allows the remoter bases to attach their own
+      # custom configuration tasks to the configuration process
+      def custom_configure_tasks_for(a=nil)
+        []
+      end
+      
+    end
+    
+  end
+end
+
+Dir["#{File.dirname(__FILE__)}/remote_bases/*.rb"].each {|base| require base }

data/lib/poolparty/plugins/gem_package.rb

@@ -0,0 +1,39 @@
+module PoolParty    
+  class Gem
+        
+    define_resource(:gem_package) do
+      
+      def has_gem_package(opts={})
+        call_function <<-EOE
+gem_package { "#{opts[:package] || opts[:name]}":
+  source => "#{opts[:source] || "http://gems.github.com" }",
+  package => "#{opts[:package] || opts[:name]}"
+}
+        EOE
+      end
+      
+      custom_function <<-EOF
+      define gem_package ($source = "http://gems", $version, $package=false) {
+        include ruby
+
+        if $version {
+          exec { "gem-package-$package":
+            path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin",
+            cwd => "/tmp",
+            command => "gem install --source $source --version \"$version\" $package",
+            unless => "gem list --local $package | grep \"$package\" | grep \"$version\""
+          }
+        } else {
+          exec { "gem-package-$package":
+            path => "/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/var/lib/gems/1.8/bin",
+            cwd => "/tmp",
+            command => "gem install --source $source $package",
+            unless => "gem list --local $package | grep \"$package\"
+          }
+        }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/plugins/line.rb

@@ -0,0 +1,76 @@
+module PoolParty    
+  class Line
+        
+    define_resource(:line_in_file) do
+      # Should refactor
+      def has_line_in_file(line="line_in_file", file="file", opts={})
+        call_function <<-EOE
+        line {
+          '#{file}_line' :
+            file => '#{file}',
+            line => '#{line}',
+            #{opts.flush_out("\t", ",")}
+            ensure => 'present'
+        }
+        EOE
+      end
+      
+      def append_if_no_such_line(line="line", file="", refreshonly='false')
+        call_function <<-EOE
+        append_if_no_such_line {
+          '#{file}' :
+            file => '#{file}',
+            line => '#{line}',
+            refreshonly => #{refreshonly}
+        }
+        EOE
+      end
+      
+      def delete_lines_from(file="", pattern=//)
+        call_function <<-EOE
+        delete_lines {
+          '#{file}' :
+            file => '#{file}',
+            pattern => '#{pattern}'
+        }
+        EOE
+      end
+      
+                    
+      custom_function <<-EOF
+      define line($file, $line, $ensure = 'present', $notify=[]) {
+        case $ensure {
+          default: { err ( "unknown ensure value ${ensure}" ) }
+          present: {
+            exec {
+              "/usr/bin/env echo '${line}' >> '${file}'": unless => "/usr/bin/env grep -qFx '${line}' '${file}'",
+               notify => $notify
+              }
+            }
+          absent: {
+            exec {
+              "/usr/bin/env sed -i '' -e '/^${line}\$/d' '${file}'": onlyif => "/usr/bin/env grep -qFx '${line}' '${file}'",
+              notify => $notify
+            }
+          }
+        }
+      }
+      define append_if_no_such_line($file, $line, $refreshonly = 'false') {
+         exec { "/bin/echo '$line' >> '$file'":
+            unless      => "/bin/grep -Fxqe '$line' '$file'",
+            path        => "/bin",
+            refreshonly => $refreshonly,
+         }
+      }
+
+      define delete_lines($file, $pattern) {
+         exec { "sed -i -r -e '/$pattern/d' $file":
+            path   => "/bin",
+            onlyif => "/bin/grep -E '$pattern' '$file'",
+         }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/plugins/svn.rb

@@ -0,0 +1,48 @@
+module PoolParty    
+  class Svn
+        
+    define_resource(:svn) do
+      
+      def has_svnpath(opts={})
+        call_function <<-EOE
+        svnserve { #{opts[:name]}:
+          source => "#{opts[:source]}",
+          path => "#{opts[:path]}",
+          user => "#{opts[:user] || false}",
+          password => "#{opts[:password] || ""}"
+        }
+        EOE
+      end
+      
+      custom_function <<-EOF
+      # Serve subversion-based code from a local location.  The job of this
+      # module is to check the data out from subversion and keep it up to
+      # date, especially useful for providing data to your Puppet server.
+      define svnserve($source, $path, $user = false, $password = false) {
+          file { $path:
+              ensure => directory,
+              owner => root,
+              group => root
+          }
+          $svncmd = $user ? {
+              false => "/usr/bin/svn co --non-interactive $source/$name .",
+              default => "/usr/bin/svn co --non-interactive --username $user --password '$password' $source/$name ."
+          }   
+          exec { "svnco-$name":
+              command => $svncmd,
+              cwd => $path,
+              require => File[$path],
+              creates => "$path/.svn"
+          }
+          exec { "svnupdate-$name":
+              command => "/usr/bin/svn update",
+              require => Exec["svnco-$name"],
+              onlyif => '/usr/bin/svn status -u --non-interactive | /bin/grep "\*"',
+              cwd => $path
+          }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/pool/base.rb

@@ -0,0 +1,74 @@
+=begin rdoc
+  Base
+  This handles user interaction, loading the parameters, etc.
+=end
+require "open-uri"
+require "ftools"
+module PoolParty
+  class Base
+    include Configurable
+    extend MethodMissingSugar
+    
+    default_options({
+      :environment => "production",
+      :user => "root", # This should change here
+      :base_keypair_path => "~/.ec2",
+      :tmp_path => "tmp",
+      :remote_storage_path => "/var/poolparty",
+      :storage_directory => File.join(Dir.pwd, "tmp"),
+      :fileserver_base => "puppet:///files",
+      :base_config_directory => "/etc/poolparty",
+      :template_directory => "tmp/templates",
+      :template_path => "/var/lib/puppet/templates",
+      :port => "80",
+      :forwarding_port => "8080",
+      :proxy_mode => "http",
+      :pool_logger_location => File.join(Dir.pwd, "logs"),
+      # EC2 Options
+      :ami => "ami-4bb05422" 
+    })
+        
+    # Class methods
+    class << self
+      def actionable_default_options
+        default_options.merge!({:access_key => self.access_key,:secret_access_key => secret_access_key})
+      end
+      def options(h={})
+        @options ||= actionable_default_options.merge(h)
+      end
+      # Get the access_key
+      def access_key
+        ENV["AWS_ACCESS_KEY_ID"] ? ENV["AWS_ACCESS_KEY_ID"] : load_keys_from_file[:access_key]
+      end
+      def secret_access_key
+        ENV["AWS_SECRET_ACCESS_ID"] ? ENV["AWS_SECRET_ACCESS_ID"] : load_keys_from_file[:secret_access_key]
+      end
+      def load_keys_from_file
+        @keys ||= get_working_key_file_locations ? YAML::load( open(get_working_key_file_locations).read ) : {}
+      end
+      def store_keys_in_file
+        unless access_key.nil? || secret_access_key.nil?
+          write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
+        end
+      end
+      def reset!
+        @keys = nil
+      end
+      # Get the instance first instance file that exists on the system from the expected places
+      # denoted in the local_instances_list_file_locations
+      def get_working_key_file_locations
+        key_file_locations.reject {|f| f unless ::File.file?(f) }.first
+      end
+      # Expected places for the instances.list to be located at on the machine
+      def key_file_locations
+        [
+          "#{Base.storage_directory}/.ppkeys",
+          "#{Base.base_config_directory}/.ppkeys",
+          "~/.ppkeys",
+          "ppkeys"
+        ]
+      end
+      
+    end
+  end    
+end

data/lib/poolparty/pool/cloud.rb

@@ -0,0 +1,132 @@
+require File.dirname(__FILE__) + "/plugin_model"
+require File.dirname(__FILE__) + "/resource"
+
+module PoolParty    
+  module Cloud
+    def cloud(name=:main, &block)
+      clouds.has_key?(name) ? clouds[name] : (clouds[name] = Cloud.new(name, self, &block))
+    end
+
+    def clouds
+      $clouds ||= {}
+    end
+    
+    def with_cloud(cl, opts={}, &block)
+      cl.options.merge!(opts)
+      cl.instance_eval &block if block
+    end
+    
+    class Cloud
+      attr_reader :name, :templates
+      include PoolParty::PluginModel
+      include PoolParty::Resources      
+      include PrettyPrinter
+      include Configurable
+      include CloudResourcer
+      extend CloudResourcer
+      # Net methods
+      include PoolParty::Remote::RemoterBase
+      include Remote
+      
+      default_options({
+        :minimum_instances => 2,
+        :maximum_instances => 4,
+        :access_key => Base.access_key,
+        :secret_access_key => Base.secret_access_key,
+        :ec2_dir => ENV["EC2_HOME"],
+        :keypair => (ENV["KEYPAIR_NAME"].nil? || ENV["KEYPAIR_NAME"].empty?) ? nil : ENV["KEYPAIR_NAME"],
+        :ami => 'ami-44bd592d',
+        :polling_time => "30.seconds"
+      })
+      
+      def initialize(name, parent, &block)
+        @name = name        
+        set_parent(parent) if parent
+        self.instance_eval &block if block_given?
+        # this can be overridden in the spec, but ec2 is the default
+        using :ec2
+      end
+                        
+      # Keypairs
+      # If the parent (pool) doesn't have a keypair defined on it, then generate one based on the 
+      # pool_name and the cloud_name
+      def keypair(*args)
+        has_keypair? ? options[:keypair] : generate_keypair(*args)
+      end
+      def has_keypair?
+        options.has_key?(:keypair) && options[:keypair]
+      end
+      def generate_keypair(*args)
+        options[:keypair] = args.length > 0 ? args[0] : "#{@parent.respond_to?(:name) ? @parent.name : ""}_#{@name}"
+      end
+      
+      def prepare_to_configuration        
+        # clear_base_directory
+        make_base_directory
+        copy_misc_templates
+        Base.store_keys_in_file
+      end
+      
+      def build_and_store_new_config_file
+        @manifest = build_manifest
+        config_file = ::File.join(Base.storage_directory, "poolparty.pp")
+        ::File.open(config_file, "w+") do |file|
+          file << "class poolparty {"
+          file << @manifest
+          file << "}"
+        end
+      end      
+      
+      def copy_misc_templates
+        ["fileserver.conf", "namespaceauth.conf"].each do |f|
+          copy_file_to_storage_directory(::File.join(::File.dirname(__FILE__), "..", "templates", f))
+        end
+      end
+      
+      # Configuration files
+      def build_manifest
+        reset_resources!        
+        add_poolparty_base_requirements
+        
+        returning Array.new do |str|
+          
+          str << resources_string_from_resources(resources)
+          
+          # Refactor this into the resources method
+          # TODO
+          services.each do |service|
+            @cp = classpackage_with_self(service)
+            str << @cp.to_string
+            str << @cp.include_string
+          end
+          
+          str << "# Custom functions"
+          str << Resources::CustomResource.custom_functions_to_string
+        end.join("\n")
+      end
+      
+      # Add all the poolparty requirements here
+      # NOTE: These are written as plugins in the lib/poolparty/base_packages directory
+      # for examples. 
+      # Also note that there is no block associated. This is because we have written
+      # all that is necessary in a method called enable
+      # which is called when there is no block
+      def add_poolparty_base_requirements        
+        heartbeat
+        haproxy
+        ruby
+        poolparty
+      end
+            
+      # Add to the services pool for the manifest listing
+      def add_service(serv)
+        services << serv
+      end
+      # Container for the services
+      def services
+        @services ||= []
+      end
+            
+    end
+  end  
+end