fairchild-poolparty 1.1.3

data/lib/poolparty/plugins/runit.rb

@@ -0,0 +1,96 @@
+module PoolParty
+  class Runit
+    define_resource(:runitservice) do
+      def has_runit_service(name="runitservice", downif="", templatedir="")
+        path = copy_templates_from_templatedir(templatedir)
+        call_custom_function <<-EOC
+runit_service {
+"#{name}":
+  directory => "/etc/sv",
+  downif => "/bin/ps aux | grep -v grep | grep -q #{downif}",
+  templatedir => "#{path}";  
+}        
+        EOC
+      end
+
+      def copy_templates_from_templatedir(dir=nil)
+        raise TemplateNotFound.new("template directory given") unless dir
+        raise TemplateNotFound.new("template directory cannot be found #{dir}") unless ::File.readable?(dir)
+        copy_directory_into_template_storage_directory(dir)
+      end
+
+      custom_function <<-EOF
+define runit_service ($directory = "/etc/sv", $downif = "/bin/false", $templatedir) {
+
+  file { "$directory-$name":
+    path => "$directory/$name",
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => Class["runit"]
+  }
+
+  file { "$directory/$name/log":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory-$name"]
+  }
+
+  file { "$directory/$name/log/main":
+    ensure => directory,
+    owner => root,
+    group => root,
+    mode => 0755,
+    require => File["$directory/$name/log"]
+  }
+
+  file { "/etc/init.d/$name":
+    ensure => $lsbdistid ? {
+      'CentOS' => "/usr/local/bin/sv",
+      default => "/usr/bin/sv",
+    },
+    require => [ File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "/var/service/$name":
+    ensure => "$directory/$name",
+    require => [ File["$directory-$name"], File["$directory/$name/run"], File["$directory/$name/log/run"] ]
+  }
+
+  file { "$directory/$name/log/run":
+    content => template("$templatedir/log-run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory/$name/log"],
+    notify => Service[$name]
+  }
+
+  file { "$directory/$name/run":
+    content => template("$templatedir/run.erb"),
+    owner => root,
+    group => root,
+    mode => 755,
+    require => File["$directory-$name"],
+    notify => Service[$name]
+  }
+  
+  service { "$name":
+    hasrestart => true,
+    hasstatus => true,
+    require => File["/etc/init.d/$name"]
+  }
+
+  exec { "$name-down":
+    command => "/etc/init.d/$name down",
+    onlyif => $downif,
+    require => File["/etc/init.d/$name"]
+  }
+}      
+      EOF
+    end    
+  end
+end

data/lib/poolparty/plugins/svn.rb

@@ -0,0 +1,48 @@
+module PoolParty    
+  class Svn
+        
+    define_resource(:svn) do
+      
+      def has_svnpath(opts={})
+        call_custom_function <<-EOE
+        svnserve { #{opts[:name]}:
+          source => "#{opts[:source]}",
+          path => "#{opts[:path]}",
+          user => "#{opts[:user] || false}",
+          password => "#{opts[:password] || ""}"
+        }
+        EOE
+      end
+      
+      custom_function <<-EOF
+      # Serve subversion-based code from a local location.  The job of this
+      # module is to check the data out from subversion and keep it up to
+      # date, especially useful for providing data to your Puppet server.
+      define svnserve($source, $path, $user = false, $password = false) {
+          file { $path:
+              ensure => directory,
+              owner => root,
+              group => root
+          }
+          $svncmd = $user ? {
+              false => "/usr/bin/svn co --non-interactive $source/$name .",
+              default => "/usr/bin/svn co --non-interactive --username $user --password '$password' $source/$name ."
+          }   
+          exec { "svnco-$name":
+              command => $svncmd,
+              cwd => $path,
+              require => File[$path],
+              creates => "$path/.svn"
+          }
+          exec { "svnupdate-$name":
+              command => "/usr/bin/svn update",
+              require => Exec["svnco-$name"],
+              onlyif => '/usr/bin/svn status -u --non-interactive | /bin/grep "\*"',
+              cwd => $path
+          }
+      }
+      EOF
+    end
+    
+  end
+end

data/lib/poolparty/poolparty/cloud.rb

@@ -0,0 +1,216 @@
+require File.dirname(__FILE__) + "/plugin_model"
+require File.dirname(__FILE__) + "/resource"
+
+module PoolParty    
+  module Cloud
+    def cloud(name=:app, &block)
+      clouds[name] ||= Cloud.new(name, &block)
+    end
+
+    def clouds
+      $clouds ||= {}
+    end
+    
+    def with_cloud(cl, opts={}, &block)
+      raise CloudNotFoundException.new("Cloud not found") unless cl
+      cl.options.merge!(opts) if opts
+      cl.run_in_context &block if block
+    end
+    
+    class Cloud < PoolParty::PoolPartyBaseClass
+      attr_reader :templates, :cloud_name, :remote_base
+
+      include CloudResourcer
+      include PoolParty::PluginModel
+      include PoolParty::Resources      
+      include PoolParty::DependencyResolverCloudExtensions
+      include PrettyPrinter
+
+      # Net methods
+      include ::PoolParty::Remote
+      include PoolParty::CloudDsl
+      # include PoolParty::Monitors
+
+      def verbose
+        true
+      end
+      
+      def self.immutable_methods
+        [:name]
+      end
+      
+      def self.method_added sym        
+        raise "Exception: #{sym.to_s.capitalize} method has been redefined" if immutable_methods.include?(sym) && !respond_to?(sym)
+      end      
+      
+      alias :name :cloud_name
+      
+      # Call the remoter commands on the cloud if they don't exist on the cloud itself
+      # This gives the cloud access to the remote_base's methods
+      def method_missing(m, *args, &block)
+        remote_base.respond_to?(m) ? remote_base.send(m, *args, &block) : super
+      end
+      
+      default_options(
+        :minimum_instances => 2,
+        :maximum_instances => 5,
+        :access_key => Default.access_key,
+        :secret_access_key => Default.secret_access_key,
+        :ec2_dir => ENV["EC2_HOME"],
+        :minimum_runtime => Default.minimum_runtime,
+        :user => Default.user
+      )
+      
+      # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
+      # call and run instance_eval on the block and then call the after_create callback
+      def initialize(name, &block)
+        @cloud_name = name
+        @cloud_name.freeze
+        plugin_directory "#{pool_specfile ? ::File.dirname(pool_specfile) : Dir.pwd}/plugins"
+        super        
+        
+        after_create
+      end
+      
+      # Fetch the name of the cloud
+      def name(*args)
+        @cloud_name ||= @cloud_name ? @cloud_name : (args.empty? ? :default_cloud : args.first)
+      end
+      
+      # Callback
+      # called after the cloud has been created, everything has run and is set at this point
+      # here the base requirements are added as well as an empty chef recipe is called
+      # Also, the after_create hook on the plugins used by the cloud are called here
+      def after_create
+        dputs "In after create"
+        ::FileUtils.mkdir_p("#{Default.tmp_path}/dr_configure")
+        run_in_context do
+          add_poolparty_base_requirements
+          chef do
+          end
+        end
+        plugin_store.each {|a| a.after_create }
+        setup_defaults
+      end
+      
+      # setup defaults for the cloud
+      def setup_defaults
+        # this can be overridden in the spec, but ec2 is the default
+        using :ec2
+        options[:keypair] ||= keypair.basename rescue nil
+        options[:rules] = {:expand => expand_when, :contract => contract_when}
+        dependency_resolver 'chef'
+      end
+      
+      # provide list of public ips to get into the cloud
+      def ips
+        list_of_running_instances.map {|ri| ri.ip }
+      end
+      
+      # TODO: make this be a random ip, since we should not rely on it being the same each time
+      def ip
+        ips.first
+      end
+            
+      # Build the new poolparty manifest
+      # Wrapping all of these requirements into the one 
+      # poolparty class.
+      # 
+      # TODO: Consider the benefits of moving all the manifest
+      # classes to separate files and keeping the containing
+      # references in the include
+      def build_and_store_new_config_file(filepath=nil, force=false)
+        filepath ||= ::File.join(Default.storage_directory, "poolparty.pp")
+        # write_properties_hash if debugging
+        vputs "Building new manifest configuration file (forced: #{force})"
+        manifest = force ? rebuild_manifest : build_manifest
+        ::File.open(filepath, "w") do |file|
+          file << manifest
+        end
+      end
+      
+      # If there is a directory named monitors in the same directory
+      # as the pool specification file is in,
+      # then create a monitors directory in the storage directory
+      # and mirror the two. When PoolParty "boots" up, it scans
+      # the monitors directory for any custom monitors
+      # that are in known locations, these are included
+      def copy_custom_monitors
+        unless Default.custom_monitor_directories.empty?
+          Default.custom_monitor_directories.each do |dir|
+            Dir["#{dir}/*.rb"].each {|f| ::Suitcase::Zipper.add(f, "monitors")}
+          end
+        end
+      end
+
+      #FIXME MOVE TO DEPENDECY RESOL
+      # Configuration files
+      def build_manifest
+        vputs "Building manifest"
+        @build_manifest ||= build_from_existing_file
+        unless @build_manifest          
+          props = to_properties_hash
+         
+          @build_manifest =  options[:dependency_resolver].send(:compile, props, self)
+        end
+        @build_manifest
+      end
+      
+      def rebuild_manifest
+        @build_manifest = nil
+        build_manifest
+      end
+      
+      # If the 
+      def build_from_existing_file
+        ::FileTest.file?("#{Default.base_config_directory}/poolparty.pp") ? open("#{Default.base_config_directory}/poolparty.pp").read : nil
+      end
+      
+      def write_properties_hash(filename=::File.join(Default.tmp_path, Default.properties_hash_filename) )
+        file_path = ::File.dirname(filename)
+        file_name = "#{::File.basename(filename, ::File.extname(filename))}_#{name}#{::File.extname(filename)}"
+        output = to_properties_hash.to_json
+        ::File.open("#{file_path}/#{file_name}", "w") {|f| f.write output }
+        true
+      end
+            
+      # Callbacks on bootstrap and configuration
+      %w( before_bootstrap 
+          after_bootstrap 
+          before_configure 
+          after_configure).each do |meth|
+        module_eval <<-EOE
+          def call_#{meth}_callbacks
+            plugin_store.each {|a| a.#{meth} }
+          end
+        EOE
+      end
+      
+      # Add all the poolparty requirements here
+      # NOTE: These are written as plugins in the lib/poolparty/base_packages directory
+      # for examples. 
+      # Also note that there is no block associated. This is because we have written
+      # all that is necessary in a method called enable
+      # which is called when there is no block
+      def add_poolparty_base_requirements
+        poolparty_base_haproxy
+        poolparty_base_heartbeat
+        poolparty_base_ruby
+        poolparty_base_packages
+      end
+      
+      def other_clouds
+        arr = []
+        clouds.each do |name, cl|
+          arr << cl if name != self.name
+        end
+        arr
+      end
+      
+      def reset!
+        reset_remoter_base!
+        @build_manifest = @describe_instances = @remote_instances_list = nil
+      end            
+    end
+  end 
+end

data/lib/poolparty/poolparty/custom_resource.rb

@@ -0,0 +1,45 @@
+require File.dirname(__FILE__) + "/resource"
+
+module PoolParty
+  def available_custom_resources
+    $available_custom_resources ||= []
+  end
+
+  module Resources
+    
+    def call_custom_function(str, opts={}, &block)
+      add_resource(:call_function, opts.merge({:str => str, :name => str.keyerize}), &block)
+    end
+                
+    # Resources for function call
+    class CallFunction < Resource
+      def to_string(pre="")
+        returning Array.new do |arr|
+          arr << "#{pre}#{str}"
+        end.join("\n")
+      end
+    end
+        
+    class CustomResource < Resource
+      def self.inherited(subclass)
+        PoolParty::Resources.available_custom_resources << subclass
+        super
+      end
+      
+      def to_string(pre="")
+        returning Array.new do |output|
+          output << "#{pre} # Custom Functions\n"
+          output << self.class.custom_functions_to_string(pre)
+        end.join("\n")        
+      end
+    end
+    
+    # Stub methods
+    # TODO: Find a better solution
+    def custom_function(*args, &block)
+    end
+    def self.custom_function(*args, &block)
+    end      
+    
+  end    
+end

data/lib/poolparty/poolparty/default.rb

@@ -0,0 +1,167 @@
+=begin rdoc
+  Base
+  This handles user interaction, loading the parameters, etc.
+=end
+require "open-uri"
+require "ftools"
+module PoolParty
+  class Default
+    include Dslify
+    
+    # def self.options
+    default_options(
+      :testing => false,
+      :debugging => false,
+      :minimum_instances => 2,
+      :maximum_instances => 5,
+      :user => "root", # This should change here
+      :base_keypair_path => "#{ENV["HOME"]}/.ec2",
+      :base_ssh_path => "#{ENV["HOME"]}/.ssh",
+      :tmp_path => "/tmp/poolparty",
+      :poolparty_home_path => "#{ENV["HOME"]}/.poolparty",
+      :remote_storage_path => "/var/poolparty",
+      :remote_gem_path => "/var/poolparty/gems",
+      :fileserver_base => "puppet://master/files",
+      :base_config_directory => "/etc/poolparty",
+      :template_directory => "templates",
+      :template_path => "/var/lib/puppet/templates",
+      :module_path => "/etc/puppet/modules/poolparty",
+      :default_specfile_name => "clouds.rb",
+      :properties_hash_filename => "clouds.json",
+      :vendor_path => "#{::File.dirname(__FILE__)}/../../../vendor",
+      :port => "80",
+      :forwarding_port => "8080",
+      :monitor_port => 8081,
+      :proxy_mode => "http",
+      :messenger_client_port => 7050,
+      :butterfly_port => 8642,
+      :minimum_runtime  => 3000, #50.minutes in seconds
+      :contract_when => "load < 0.25",
+      :expand_when => "load > 0.9",
+      # :agent_pid_file => ::File.readable?("/var/run/poolparty_agent.pid") ? "/var/run/agent.pid" : "#{Dir.pwd}/agent.pid",
+      # :agent_port => 8081,
+      # EC2 Options
+      :ami => "ami-7cfd1a15",
+      :size => 'm1.small', # must be 'm1.small', 'm1.large', 'm1.xlarge', 'c1.medium', or 'c1.xlarge'  #TODO: change to instance_size
+      :availabilty_zone => "us-east-1a",
+      :security_group => ["default"],
+      # Options that should not be touched pretty much ever
+      :manifest_path => "/etc/puppet/manifests" 
+    )
+    
+    def options
+      default_options
+    end
+    
+    # Class methods
+    class << self
+      def method_missing(m,*a,&block)
+        default_options.include?(m) ? default_options[m] : super
+      end
+      def options
+        default_options
+      end
+      # Get the access_key
+      def access_key
+        @access_key ||= load_access_keys_from_environment_var || load_keys_from_file[:access_key]
+      end
+      def load_access_keys_from_environment_var
+        [ ENV["AWS_ACCESS_KEY"], ENV["AWS_ACCESS_KEY_ID"]].reject {|a| a.nil? }.first
+      end
+      def secret_access_key
+        @secret_access_key ||= load_secret_access_keys_from_environment_var || load_keys_from_file[:secret_access_key]
+      end
+      def load_secret_access_keys_from_environment_var
+        [ ENV["AWS_SECRET_ACCESS_KEY"] ].reject {|a| a.nil? }.first
+      end
+      def read_keyfile
+        open(get_working_key_file_locations).read
+      end
+      def load_keys_from_file
+        @keys ||= get_working_key_file_locations ? YAML::load( read_keyfile ) : {}
+      end
+      # Store the keys in a yaml format to give the master access
+      # So that the master has access to the files
+      def store_keys_in_file
+        unless access_key.nil? || secret_access_key.nil?
+          write_to_file( key_file_locations.first, YAML::dump({:access_key => access_key, :secret_access_key => secret_access_key}))        
+        end
+      end
+      def store_keys_in_file_for(obj=nil)
+        if obj
+          @access_key = obj.access_key
+          @secret_access_key = obj.secret_access_key
+        end
+        store_keys_in_file
+      end
+      def reset!
+        @keys = nil
+      end
+      # Get the instance first instance file that exists on the system from the expected places
+      # denoted in the local_instances_list_file_locations
+      def get_working_key_file_locations
+        key_file_locations.reject {|f| f unless ::File.file?(f) }.first
+      end
+      # Expected places for the instances.list to be located at on the machine
+      def key_file_locations
+        [
+          ".ppkeys",
+          "#{Default.base_config_directory}/.ppkeys",
+          "#{Default.storage_directory}/ppkeys",          
+          "~/.ppkeys",
+          "ppkeys"
+        ]
+      end
+      def properties_hash_file
+        [
+          Default.base_config_directory,
+          Dir.pwd
+        ].collect do |dir|
+          full_dir = ::File.join(dir, Default.properties_hash_filename)
+          full_dir if ::File.file?(full_dir)
+        end.compact.first || "#{Default.base_config_directory}/#{Default.properties_hash_filename}"
+      end
+      def storage_directory
+        [
+            "/var/poolparty"
+        ].select do |dir|
+          dir if viable_directory?(dir)
+        end.first || ::File.join( "/tmp/poolparty")
+      end
+      def logger_location
+        [
+            "/var/log/poolparty"
+        ].select do |dir|
+          dir if viable_directory?(dir)
+        end.first
+      end
+      # Assume the logs will be at the pool.log location within the 
+      # logger_location set above
+      def pool_logger_location
+        ::File.join(logger_location, "poolparty.log")
+      end
+      def custom_monitor_directories
+        [
+          "/var/poolparty/monitors",
+          "/etc/poolparty/monitors",
+          "#{Dir.pwd}/monitors"
+        ].select {|d| d if viable_directory?(d) }
+      end
+      
+      def custom_modules_directories
+        [
+          "/var/poolparty/modules",
+          "/etc/poolparty/modules",
+          "#{Dir.pwd}/modules"
+        ].select {|d| d if viable_directory?(d) }
+      end
+      # Only return true if the directory we are reading is both readable
+      # and exists
+      def viable_directory?(dir)
+        ::File.directory?(dir) && ::File.readable?(dir)
+      end
+    end
+    #end of class methods
+    
+  end    
+end

data/lib/poolparty/poolparty/key.rb

@@ -0,0 +1,75 @@
+# STUB FOR NOW
+# TODO
+module PoolParty
+  class Key
+    
+    attr_accessor :filepath
+    
+    # Create a new key that defaults to id_rsa as the name. 
+    def initialize(filepath=nil)
+      @filepath = (filepath.nil? || filepath.empty?) ? "id_rsa" : filepath
+    end
+    
+    # If the full_filepath is nil, then the key doesn't exist
+    def exists?
+      full_filepath != nil
+    end
+    
+    # Read the content of the key
+    def content
+      @content ||= exists? ? open(full_filepath).read : nil
+    end
+        
+    # Returns the full_filepath of the key. If a full filepath is passed, we just return the expanded filepath
+    # for the keypair, otherwise query where it is against known locations
+    def full_filepath
+      @full_filepath ||= ::File.file?(::File.expand_path(filepath)) ? ::File.expand_path(filepath) : search_in_known_locations
+    end
+    alias :to_s :full_filepath
+    
+    # Basename of the keypair
+    def basename
+      @basename ||= ::File.basename(full_filepath, ::File.extname(full_filepath)) rescue filepath
+    end
+    
+    # Just the filename of the keypair
+    def filename
+      @filename ||= ::File.basename(full_filepath) rescue filepath
+    end
+    
+    # Search for the key in default locations with the entire filepath
+    # if the file exists. If it doesn't exist in the default locations, 
+    # then it returns nil and assumes we it doesn't exist
+    def search_in_known_locations
+      self.class.keypair_paths.each do |path|
+        full_path = ::File.join( ::File.expand_path(path), filepath)
+        return full_path if ::File.exists?(full_path)
+      end
+      nil
+    end
+    
+    # Default locations to search for the key
+    def self.keypair_paths
+      [ "#{ENV["HOME"]}/.ssh",
+        "#{Default.poolparty_home_path}/keys",
+        Default.base_keypair_path,
+        Default.base_config_directory,
+        Default.base_ssh_path,
+        Default.remote_storage_path,        
+        Dir.pwd
+      ]
+    end
+    
+    # Support to add the enumerable each to keys
+    def each
+      yield full_filepath
+    end
+    
+    # Turn the keypair into the a useful json string
+    def to_json
+      "{\"basename\":\"#{basename}\",
+       \"full_filepath\": \"/etc/poolparty/#{filename}\"}"
+    end
+    
+  end
+end

data/lib/poolparty/poolparty/loggable.rb

@@ -0,0 +1,28 @@
+#todo: Deprecate
+# =begin rdoc
+#   A logging class to allow us to log to locations
+# =end
+# class Loggable
+#   def initialize    
+#     self.class.loggers << file_logger
+#     file_logger.level = :info
+#   end
+#   def file_logger
+#     @file_logger ||= Logging.logger( Default.pool_logger_location, logging_opts )
+#   end
+#   def stdout_logger
+#     @stdout_logger ||= Logging.logger(STDOUT, logging_opts.merge({:pattern => "%m\n"}))
+#   end
+#   def logging_opts
+#     {:pattern => "[%d] %-l : %m\n",
+#     :date_pattern => "%Y-%m-%d %H:%M:%S.%s"}
+#   end
+#   %w(info warn debug notice).each do |meth|
+#     define_method(meth.to_sym) do |*args|
+#       self.class.loggers.each {|l| l.send meth.to_sym, args}
+#     end
+#   end
+#   def self.loggers
+#     @loggers ||= []
+#   end
+# end