auser-poolparty 0.2.81 → 0.2.84

data/lib/poolparty/provisioners/provisioner_base.rb

@@ -3,246 +3,104 @@
   This class only comes in to play when calling the setup commands on
   the development machine
 =end
+require 'capistrano/cli'
+
 module PoolParty
   module Provisioner
     
-    # Provision master
-    # Convenience method to clean 
-    def self.provision_master(cloud, testing=false)
-      Provisioner::Master.new(cloud).process_install!(testing)
-    end
-
-    def self.configure_master(cloud, testing=false)
-      Provisioner::Master.new(cloud).process_configure!(testing)
-    end
-    
-    def self.reconfigure_master(cloud, testing=false)
-      Provisioner::Master.new(cloud).process_reconfigure!(testing)
-    end
-
-    def self.provision_slaves(cloud, testing=false)
-      cloud.nonmaster_nonterminated_instances.each do |sl|
-        provision_slave(sl, cloud, testing)
-      end
-    end
-
-    def self.configure_slaves(cloud, testing=false)
-      cloud.nonmaster_nonterminated_instances.each do |sl|
-        configure_slave(sl, cloud, testing)
-      end
-    end
-        
-    def self.provision_slave(instance, cloud, testing=false)
-      Provisioner::Slave.new(instance, cloud).process_install!(testing)
-    end
-    
-    def self.configure_slave(instance, cloud, testing=false)
-      Provisioner::Slave.new(instance, cloud).process_configure!(testing)
-    end
-    
-    def self.become_master(cloud, testing=false)
-      Provisioner::BecomeMaster.new(cloud).process_install!(testing)
-    end
-    
-    def self.process_clean_reconfigure_for!(instance, cloud, testing=false)
-      Provisioner::Master.new(cloud).process_clean_reconfigure_for!(instance, testing)
-    end
-    
-    def self.clear_master_ssl_certs(cloud, testing=false)
-      Provisioner::Master.new(cloud).clear_master_ssl_certs
+    def provisioner_for(inst)
+      PoolParty::Provisioner::Capistrano.new(inst, self, :ubuntu)      
     end
     
     class ProvisionerBase
+      attr_accessor :config, :loaded_tasks, :instance, :cloud, :os
       
       include Configurable
       include CloudResourcer
       include FileWriter
       
-      def initialize(instance,cld=self, os=:ubuntu)
+      def initialize(instance=nil, cld=self, os=:ubuntu, &block)
         @instance = instance
         @cloud = cld
         
         options(cloud.options) if cloud && cloud.respond_to?(:options)
-        set_vars_from_options(instance.options) unless instance.nil? || !instance.options || !instance.options.empty?
-        options(instance.options) if instance.respond_to?(:options)
+        # set_vars_from_options(instance.options) unless instance.nil? || !instance.options || !instance.options.empty?
+        # options(instance.options) if instance.respond_to?(:options)
+        
+        @os = os.to_s.downcase.to_sym        
+        self.instance_eval &block if block
         
-        @os = os.to_s.downcase.to_sym
         loaded
       end
+      
+      def provision_master?
+        !@instance.nil? && @instance.master?
+      end
+
       # Callback after initialized
-      def loaded(opts={}, parent=self)      
+      def loaded
+      end
+      
+      def loaded_tasks
+        @loaded_tasks ||= []
       end
       
       ### Installation tasks
       
       # This is the actual runner for the installation    
-      def install
-        valid? ? install_string : error
-      end
-      # Write the installation tasks to a file in the storage directory
-      def write_install_file
-        error unless valid?
-        ::FileUtils.mkdir_p Base.storage_directory unless ::File.exists?(Base.storage_directory)
-        provisioner_file = ::File.join(Base.storage_directory, "install_#{name}.sh")
-        ::File.open(provisioner_file, "w+") {|f| f << install }
-      end
-      def name
-        @instance.name
-      end
-      # TODO: Clean up this method
-      def process_install!(testing=false)
+      def install(testing=false)
         error unless valid?
-        write_install_file
         setup_runner
-        
-        unless testing
-          vputs "Logging on to #{@instance.ip} (#{@instance.name})"
-          @cloud.rsync_storage_files_to(@instance)
-          vputs "Preparing configuration on the master"
-          
+        unless testing          
           before_install(@instance)
-          
-          # process_clean_reconfigure_for!(@instance, testing)
-          
-          vputs "Logging in and running provisioning on #{@instance.name}"
-          # /bin/rm install_#{name}.sh
-          cmd = "cd #{Base.remote_storage_path} && /bin/chmod +x install_#{name}.sh && /bin/sh install_#{name}.sh"
-          verbose ? @cloud.run_command_on(cmd, @instance) : hide_output {@cloud.run_command_on(cmd, @instance)}
-          
-          process_clean_reconfigure_for!(@instance, testing)
-          
+
+          vputs "Provisioning #{@instance.name}"
+          process_install!(testing)
+
           after_install(@instance)
         end
       end
-      # Install callbacks
-      # Before installation callback
-      def before_install(instance)        
-      end
-      def after_install(instance)        
+      # The provisioner bases overwrite this method
+      def process_install!(testing=false)
       end
       
-      ### Configuraton tasks
-      
-      def configure
-        valid? ? configure_string : error
-      end
-      def write_configure_file
-        error unless valid?
-        provisioner_file = ::File.join(Base.storage_directory, "configure_#{name}.sh")
-        ::File.open(provisioner_file, "w+") {|f| f << configure }
-      end
-      def process_configure!(testing=false)
+      # Configuration
+      def configure(testing=false)
         error unless valid?
-        write_configure_file
         setup_runner
-        
-        unless testing
-          vputs "Logging on to #{@instance.ip}"
-          @cloud.rsync_storage_files_to(@instance)
-          #  && /bin/rm configure_#{name}.sh
-          cmd = "cd #{Base.remote_storage_path} && /bin/chmod +x configure_#{name}.sh && /bin/sh configure_#{name}.sh"
-          verbose ? @cloud.run_command_on(cmd, @instance) : hide_output {@cloud.run_command_on(cmd, @instance)}
-        end
-      end
-      def process_clean_reconfigure_for!(instance, testing=false)
-        if instance.is_a?(String)
-          name = instance
-          instance = MyOpenStruct.new(:name => name)
-        end
-        vputs "Cleaning certs from master: #{instance.name}"
-        # puppetca --clean #{instance.name}.compute-1.internal; puppetca --clean #{instance.name}.ec2.internal
-        # find /etc/puppet/ssl -type f -exec rm {} \;
-        unless testing
-          # @cloud.run_command_on("rm -rf /etc/puppet/ssl", instance) unless instance.master?
-          str = returning String.new do |s|
-            s << "puppetca --clean #{instance.name}.compute-1.internal 2>&1 > /dev/null;"
-            s << "puppetca --clean #{instance.name}.ec2.internal 2>&1 > /dev/null"
-          end
-          # @cloud.run_command_on(str, @cloud.master)
-        end
-      end
-      def clear_master_ssl_certs
-        str = returning String.new do |s|
-          s << "puppetca --clean master.compute-1.internal 2>&1 > /dev/null;"
-          s << "puppetca --clean master.ec2.internal 2>&1 > /dev/null"
+        unless testing          
+          before_configure(@instance)
+
+          vputs "Provisioning #{@instance.name}"
+          process_configure!(testing)
+
+          after_configure(@instance)
         end
-        @cloud.run_command_on("if [ -f '/usr/bin/puppetcleaner' ]; then /usr/bin/env puppetcleaner `hostname`; else #{str}; fi", @cloud.master)
-      end
-      def process_reconfigure!(testing=false)
-        @cloud.run_command_on(PoolParty::Remote::RemoteInstance.puppet_runner_command, @instance) unless testing
       end
+      
       # Tasks that need to be performed everytime we do any
       # remote ssh'ing into any instance
       def setup_runner(force=false)
         @cloud.prepare_for_configuration
         @cloud.build_and_store_new_config_file(force)
       end
-      def valid?
-        true
-      end
-      def error
-        "Error in installation"
-      end
-      # Gather all the tasks into one string
-      def install_string
-        [default_install_tasks, last_install_tasks].flatten.each do |task|
-          case task.class
-          when String
-            task
-          when Method
-            self.send(task.to_sym)
-          end
-        end.nice_runnable
-      end
-      def last_install_tasks
-        []
-      end
-      def configure_string
-        [default_configure_tasks, last_configure_tasks].flatten.each do |task|
-          case task.class
-          when String
-            task
-          when Method
-            self.send(task.to_sym)
-          end
-        end.nice_runnable
-      end
-      def last_configure_tasks
-        []
+      
+      # Callbacks
+      # Before installation callback
+      def before_install(instance)        
       end
-      # Tasks with default tasks 
-      # These are run on all the provisioners, master or slave
-      def default_install_tasks
-        [
-          "#!/usr/bin/env sh",
-          first_install_tasks,
-          upgrade_system,
-          install_rubygems,
-          make_logger_directory,
-          install_puppet,
-          fix_rubygems,
-          setup_system_for_poolparty,
-          custom_install_tasks
-        ] << install_tasks
+      def after_install(instance)        
       end
-      # Tasks with default configuration tasks
-      # This is run on the provisioner, regardless
-      def default_configure_tasks
-        [
-          custom_configure_tasks
-        ] << configure_tasks
+      def before_configure(instance)        
       end
-      # Build a list of the tasks to run on the instance
-      def install_tasks(a=[])
-        @install_task ||= a
+      def after_configure(instance)        
       end
-      # Set the first tasks to be called on the instance
-      # before any other tasks are run
-      def first_install_tasks
-        @cloud.first_install_tasks_for(@instances) || []
+      
+      def valid?
+        true
       end
-      def configure_tasks(a=[])
-        @configure_tasks ||= a
+      def error
+        raise ProvisionerException.new("Error in installation")
       end
       # Custom installation tasks
       # Allow the remoter bases to attach their own tasks on the 
@@ -274,107 +132,73 @@ module PoolParty
           :gentoo => "emerge"
         }
       end
+      def os_installer
+        "#{self.class.installers[@os]}"
+      end
       # Convenience method to grab the installer
       def installer_for(names=[])
         packages = names.is_a?(Array) ? names.join(" ") : names
         "#{self.class.installers[@os]} #{packages}"
       end
+            
+      #TODO#
+      # Abstract the gems out
+      def base_gems
+        {
+          :logging => "http://rubyforge.org/frs/download.php/44731/logging-0.9.4.gem",
+          :ZenTest => "http://rubyforge.org/frs/download.php/45581/ZenTest-3.11.0.gem",
+          :ParseTree => "http://rubyforge.org/frs/download.php/45600/ParseTree-3.0.1.gem",
+          :ruby2ruby => "http://rubyforge.org/frs/download.php/45587/ruby2ruby-1.2.0.gem",
+          :activesupport => "http://rubyforge.org/frs/download.php/45627/activesupport-2.1.2.gem",
+          :"xml-simple" => "http://rubyforge.org/frs/download.php/18366/xml-simple-1.0.11.gem",
+          :RubyInline => "http://rubyforge.org/frs/download.php/45683/RubyInline-3.8.1.gem",
+          :flexmock => "http://rubyforge.org/frs/download.php/42580/flexmock-0.8.3.gem",
+          :hoe => "http://rubyforge.org/frs/download.php/45685/hoe-1.8.2.gem",
+          :lockfile => "http://rubyforge.org/frs/download.php/18698/lockfile-1.4.3.gem",
+          :rubyforge => "http://rubyforge.org/frs/download.php/45546/rubyforge-1.0.1.gem",
+          :rake => "http://rubyforge.org/frs/download.php/43954/rake-0.8.3.gem",
+          :sexp_processor => "http://rubyforge.org/frs/download.php/45589/sexp_processor-3.0.0.gem",
+          :capistrano => "http://rubyforge.org/frs/download.php/48031/capistrano-2.5.3.gem",
+          :poolparty => "http://github.com/auser/poolparty/tree/master%2Fpkg%2Fpoolparty.gem?raw=true",
+          "ec2" => "http://rubyforge.org/frs/download.php/43666/amazon-ec2-0.3.1.gem"
+        }
+      end
       
-      # Install from the class-level
-      def self.install(instance, cl=self)
-        new(instance, cl).install
+      def download_base_gems_string
+        returning(Array.new) do |arr|
+          base_gems.each do |name, url|
+            arr << "wget #{url} -O #{Base.remote_storage_path}/#{name}.gem 2>&1"
+          end
+        end.join(" && ")
       end
-
-      def self.configure(instance, cl=self)
-        new(instance, cl).configure
+      
+      def install_base_gems_string
+        returning(Array.new) do |arr|
+          base_gems.each do |name, url|
+            arr << "/usr/bin/gem install --ignore-dependencies --no-ri --no-rdoc #{Base.remote_storage_path}/#{name}.gem"
+          end
+        end.join(" && ")
       end
       
       # Template directory from the provisioner base
       def template_directory
         File.join(File.dirname(__FILE__), "..", "templates")
       end
-      
-      def install_rubygems
-        <<-EOE
-        #{installer_for("ruby rubygems")}
-        EOE
-      end
-      
-      def fix_rubygems        
-        <<-EOE
-          echo '#{open(::File.join(template_directory, "gem")).read}' > /usr/bin/gem
-          echo 'Updating rubygems'
-          PAT=`/usr/bin/gem env gemdir`
-          /usr/bin/gem update --system #{unix_hide_string}
-          /usr/bin/gem update --system #{unix_hide_string}
-        EOE
-      end
-
-      def create_local_node
-        str = <<-EOS
-  node default {
-    include poolparty
-  }
-        EOS
-         @cloud.list_of_running_instances.each do |ri|
-           str << <<-EOS           
-  node "#{ri.name}" {}
-           EOS
-         end
-        "echo '#{str}' > /etc/puppet/manifests/nodes/nodes.pp"
+                  
+      # Install from the class-level
+      def self.install(instance, cl=self, testing=false)
+        new(instance, cl).install(testing)
       end
-      
-      def upgrade_system
-        case @os
-        when :ubuntu
-          "
-if grep -q 'http://mirrors.kernel.org/ubuntu hardy main universe' /etc/apt/sources.list
-then 
-echo 'Updated already'
-else
-touch /etc/apt/sources.list
-echo 'deb http://mirrors.kernel.org/ubuntu hardy main universe' >> /etc/apt/sources.list
-aptitude update -y #{unix_hide_string} <<heredoc
-Y
 
-heredoc
-fi
-          "
-        else
-          "# No system upgrade needed"
-        end
-      end
-      
-      def install_puppet
-        "#{installer_for( puppet_packages )}"
-      end
-      
-      def make_logger_directory
-        "mkdir -p /var/log/poolparty"
+      def self.configure(instance, cl=self, testing=false)
+        new(instance, cl).configure(testing)
       end
       
-      def create_poolparty_manifest
-        <<-EOS
-          cp #{Base.remote_storage_path}/poolparty.pp /etc/puppet/manifests/classes
-        EOS
-      end
-      def setup_system_for_poolparty
-        <<-EOS
-          mkdir -p #{Base.base_config_directory}/ssl/private_keys
-          mkdir -p #{Base.base_config_directory}/ssl/certs
-          mkdir -p #{Base.base_config_directory}/ssl/public_keys
-          
-          cp #{Base.remote_storage_path}/#{Base.template_directory}/puppetrerun /usr/bin/puppetrerun
-          chmod +x /usr/bin/puppetrerun
-          
-          mv #{Base.remote_storage_path}/cookie ~/.erlang.cookie
-          chmod 400 ~/.erlang.cookie
-        EOS
-      end
     end
   end  
 end
+
 ## Load the provisioners
-Dir[File.dirname(__FILE__) + "/provisioners/*.rb"].each do |file|
+Dir[File.dirname(__FILE__) + "/*/*.rb"].each do |file|
   require file
 end

data/lib/poolparty/templates/gem

@@ -1,20 +1,18 @@
-#! /usr/bin/ruby1.8
+#!/usr/bin/env ruby
 #--
 # Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others.
 # All rights reserved.
 # See LICENSE.txt for permissions.
-# /usr/bin/gem is broken, so... have to use this until it is fixed
 #++
 
+require 'rubygems'
+require 'rubygems/gem_runner'
+require 'rubygems/exceptions'
 
-require "rubygems"
-require "rubygems/gem_runner"
-Gem.manage_gems
+required_version = Gem::Requirement.new "> 1.8.3"
 
-required_version = Gem::Version::Requirement.new(">= 1.8.0")
-unless  required_version.satisfied_by?(Gem::Version.new(RUBY_VERSION))
-  puts "Expected Ruby Version #{required_version}, was #{RUBY_VERSION}"
-  exit(1)
+unless required_version.satisfied_by? Gem.ruby_version then
+  abort "Expected Ruby Version #{required_version}, was #{Gem.ruby_version}"
 end
 
 # We need to preserve the original ARGV to use for passing gem options
@@ -22,4 +20,8 @@ end
 # it...its for the source building process.
 args = !ARGV.include?("--") ? ARGV.clone : ARGV[0...ARGV.index("--")]
 
-Gem::GemRunner.new.run(args)
+begin
+  Gem::GemRunner.new.run args
+rescue Gem::SystemExitException => e
+  exit e.exit_code
+end

data/lib/poolparty/version.rb

@@ -2,7 +2,7 @@ module PoolParty
   module VERSION #:nodoc:
     MAJOR = 0
     MINOR = 2
-    TINY  = 81
+    TINY  = 84
 
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/lib/poolparty.rb

@@ -2,7 +2,7 @@ require 'rubygems'
 
 # Load required gems
 @required_software = Array.new
-%w(activesupport ftools logging resolv ruby2ruby digest/sha2 open3).each do |lib|
+%w(activesupport ftools logging resolv ruby2ruby digest/sha2 open3 capistrano).each do |lib|
   begin
     require lib
   rescue Exception => e
@@ -36,7 +36,9 @@ end
 ActiveSupport::Dependencies.load_paths << File.dirname(__FILE__)
 
 ## Load PoolParty
-require "#{File.dirname(__FILE__)}/poolparty/version"
+%w(version).each do |f|
+  require "#{File.dirname(__FILE__)}/poolparty/#{f}"
+end
 
 %w(core modules exceptions dependency_resolutions aska monitors provisioners extra net).each do |dir|
   Dir[File.dirname(__FILE__) + "/poolparty/#{dir}/**.rb"].each do |file|
@@ -50,7 +52,7 @@ Logging.init :debug, :info, :warn, :error, :fatal
 module PoolParty
   include FileWriter
   
-  def logger
+  def log
     @logger ||= make_new_logger
   end
   

data/poolparty.gemspec

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: poolparty
 version: !ruby/object:Gem::Version 
-  version: 0.2.81
+  version: 0.2.84
 platform: ruby
 authors: 
 - Ari Lerner
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-12-10 00:00:00 -08:00
+date: 2008-12-13 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -70,6 +70,7 @@ executables:
 - cloud-refresh
 - cloud-rsync
 - cloud-run
+- cloud-setup-dev
 - cloud-spec
 - cloud-ssh
 - cloud-start
@@ -99,6 +100,7 @@ executables:
 - server-stop-master
 - server-stop-node
 - server-update-hosts
+- server-write-new-nodes
 extensions: []
 
 extra_rdoc_files: 
@@ -111,6 +113,7 @@ extra_rdoc_files:
 - lib/poolparty/config/postlaunchmessage.txt
 - website/index.txt
 files: 
+- Capfile
 - History.txt
 - License.txt
 - Manifest.txt
@@ -131,6 +134,7 @@ files:
 - bin/cloud-refresh
 - bin/cloud-rsync
 - bin/cloud-run
+- bin/cloud-setup-dev
 - bin/cloud-spec
 - bin/cloud-ssh
 - bin/cloud-start
@@ -160,6 +164,7 @@ files:
 - bin/server-stop-master
 - bin/server-stop-node
 - bin/server-update-hosts
+- bin/server-write-new-nodes
 - config/hoe.rb
 - config/requirements.rb
 - examples/basic.rb
@@ -286,6 +291,7 @@ files:
 - lib/poolparty/base_packages/poolparty.rb
 - lib/poolparty/base_packages/ruby.rb
 - lib/poolparty/base_packages/runit.rb
+- lib/poolparty/capistrano.rb
 - lib/poolparty/config/postlaunchmessage.txt
 - lib/poolparty/core/array.rb
 - lib/poolparty/core/class.rb
@@ -306,6 +312,7 @@ files:
 - lib/poolparty/exceptions/CloudNotFoundException.rb
 - lib/poolparty/exceptions/LoadRulesException.rb
 - lib/poolparty/exceptions/MasterException.rb
+- lib/poolparty/exceptions/ProvisionerException.rb
 - lib/poolparty/exceptions/RemoteException.rb
 - lib/poolparty/exceptions/ResourceException.rb
 - lib/poolparty/exceptions/RuntimeException.rb
@@ -374,10 +381,12 @@ files:
 - lib/poolparty/poolparty/resources/symlink.rb
 - lib/poolparty/poolparty/resources/variable.rb
 - lib/poolparty/poolparty/script.rb
+- lib/poolparty/provisioners/capistrano/capistrano.rb
+- lib/poolparty/provisioners/capistrano/capistrano_configurer.rb
+- lib/poolparty/provisioners/capistrano/recipies/base.rb
+- lib/poolparty/provisioners/capistrano/recipies/master.rb
+- lib/poolparty/provisioners/capistrano/recipies/slave.rb
 - lib/poolparty/provisioners/provisioner_base.rb
-- lib/poolparty/provisioners/provisioners/become_master.rb
-- lib/poolparty/provisioners/provisioners/master.rb
-- lib/poolparty/provisioners/provisioners/slave.rb
 - lib/poolparty/spec/core/string.rb
 - lib/poolparty/spec/matchers/a_spec_extensions_base.rb
 - lib/poolparty/spec/matchers/have_cron.rb
@@ -445,9 +454,6 @@ files:
 - spec/poolparty/helpers/binary_spec.rb
 - spec/poolparty/helpers/display_spec.rb
 - spec/poolparty/helpers/optioner_spec.rb
-- spec/poolparty/helpers/provisioner_base_spec.rb
-- spec/poolparty/helpers/provisioners/master_spec.rb
-- spec/poolparty/helpers/provisioners/slave_spec.rb
 - spec/poolparty/modules/cloud_resourcer_spec.rb
 - spec/poolparty/modules/configurable_spec.rb
 - spec/poolparty/modules/definable_resource.rb
@@ -456,6 +462,7 @@ files:
 - spec/poolparty/monitors/base_monitor_spec.rb
 - spec/poolparty/monitors/monitors/cpu_monitor_spec.rb
 - spec/poolparty/monitors/monitors/memory_monitor_spec.rb
+- spec/poolparty/net/log/pool.log
 - spec/poolparty/net/messenger_spec.rb
 - spec/poolparty/net/remote_bases/ec2_spec.rb
 - spec/poolparty/net/remote_instance_spec.rb
@@ -497,6 +504,8 @@ files:
 - spec/poolparty/poolparty/test_plugins/virtual_host_template.erb
 - spec/poolparty/poolparty/test_plugins/webserver.rb
 - spec/poolparty/poolparty_spec.rb
+- spec/poolparty/provisioners/capistrano/capistrano_spec.rb
+- spec/poolparty/provisioners/provisioner_base_spec.rb
 - spec/poolparty/spec/core/string_spec.rb
 - spec/poolparty/spec_helper.rb
 - tasks/cloud.rake
@@ -521,7 +530,7 @@ files:
 has_rdoc: true
 homepage: http://poolparty.rubyforge.org
 post_install_message: |-
-  Get ready to jump in the pool, you just installed PoolParty! (Updated at 13:44 12/10/08)
+  Get ready to jump in the pool, you just installed PoolParty! (Updated at 19:18 12/13/08)
   
   To get started, run the generator:
   

data/spec/poolparty/helpers/optioner_spec.rb

@@ -33,7 +33,7 @@ describe "Option Parser" do
       @op = PoolParty::Optioner.new(%w( -v -i 1 five six -x), {:abstract => true, :parse_options => false})
     end
     it "should have an array of default boolean args" do
-      @op.boolean_args.sort.should == ['-V', '-h', '-t', '-v']
+      @op.boolean_args.sort.should == ['--debug', '-V', '-h', '-t', '-v']
     end
     it "should split ARGV into flagged and unflagged arg arrays" do
       @op.unflagged_args.sort.sort.should ==  ["five", "six"]