auser-poolparty 1.3.5 → 1.3.6

data/Rakefile

@@ -39,8 +39,8 @@ end
 
 # Generate documentation
 Rake::RDocTask.new do |rd|
-  rd.main = "Readme.txt"
-  rd.rdoc_files.include("Readme.txt", "lib/**/*.rb")
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
   rd.rdoc_dir = "rdoc"
   # rd.template = "hanaa"
 end

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 3
-:patch: 5
+:patch: 6
 :major: 1

data/bin/cloud-misc

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require "poolparty"
+
+require 'git-style-binary/command'
+
+GitStyleBinary.command do
+  banner <<-EOS
+Usage: #{$0} #{all_options_string}
+
+  read the code 
+EOS
+
+  short_desc "various cloud development tasks"
+  
+  opt :inst_num, "The number of the instance to run bootstrap on", :type => :int
+  opt :force, "Force rebootstrapping (useful for upgrading)", :default => false
+
+  run do |command|
+    @loaded_clouds.each do |cld|
+      cld.nodes.each do |node|
+
+        case command.argv.shift
+        when "deploy_hermes"
+          hermesdir = File.dirname(__FILE__) + "/../vendor/erlang/hermes"
+          release = `#{hermesdir}/scripts/most_recent_release tar.gz`.strip
+          node.scp(:source => release, :destination => "/var/poolparty/tmp")
+          node.scp(:source => "#{hermesdir}/scripts/target_system", :destination => "/var/poolparty/tmp")
+        end
+
+      end
+    end
+  end
+end

data/bin/cloud-ssh

@@ -13,11 +13,14 @@ EOS
 
   short_desc "ssh into an instance of the cloud"
   
+  opt :instance_number, "Ssh into the instance number", :type => :integer, :default => 0
+  
   run do |command|
     cloud = @loaded_clouds.first
     
     if !cloud.nodes.empty?
-      cloud.nodes.first.ssh
+      inst = cloud.nodes[command[:instance_number]]
+      inst ? inst.ssh : puts("Error: The instance number (#{command[:instance_number]}) is too high for the current number of instances")
     else
       puts "No running instances can be found"
     end

data/config/jeweler.rb

@@ -26,11 +26,12 @@ end
 
     s.files = (%w(Rakefile README.rdoc License.txt VERSION.yml) + Dir["{config,examples,lib,test,tasks,script,generators,bin,vendor}/**/*"])
     
-    s.files.exclude '**/*.beam'
-    s.files.exclude "**/*/erl_crash.dump"
+    s.files.exclude 'vendor/erlang/*'
+    s.files.exclude 'examples/thrift/**/*.beam'
+    # s.files.exclude "**/*/erl_crash.dump"
     
     s.has_rdoc = true
-    s.extra_rdoc_files = ["README.txt", "License.txt", 'History.txt']
+    s.extra_rdoc_files = ["README.rdoc", "License.txt", 'History.txt']
     s.require_paths = ["lib"]
     s.rdoc_options = ['--quiet', '--title', 'PoolParty documentation',
         # "index.html",

data/examples/monitored_cloud.rb

@@ -10,7 +10,7 @@ pool "poolparty" do
       image_id 'emi-39CA160F'
     end
     
-    monitor :cpu do |c|
+    monitor :'cpu-idle' do |c|
       vote_for(:expand) if c > 0.9
       vote_for(:contract) if c < 0.1
     end

data/examples/thrift/thrift_example.rb

@@ -3,7 +3,7 @@
 $:.unshift(File.dirname(__FILE__) + "/../../lib")
 $:.unshift(File.dirname(__FILE__) + "/../../examples")
 require "poolparty"
-require "simple"
+# require "simple"
 
 $:.unshift("#{File.dirname(__FILE__)}/../../lib/proto/gen-rb")
 
@@ -15,6 +15,7 @@ require "poolparty_types"
 port = ARGV.pop || 11223
 
 transport = Thrift::BufferedTransport.new(Thrift::Socket.new('localhost', port))
+# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('vm', port))
 protocol = Thrift::BinaryProtocol.new(transport)
 
 client = CloudThrift::CommandInterface::Client.new(protocol)
@@ -22,6 +23,7 @@ transport.open()
 
 cld = CloudThrift::CloudQuery.new
 cld.name = 'monitored_app'
+# cld.name = 'vmware'
 
 resp = client.run_command(cld, "name", [])
 puts resp.response
@@ -29,8 +31,8 @@ puts resp.response
 resp = client.run_command(cld, "maximum_instances", [])
 puts resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.91"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.91"])
 p resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.01"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.01"])
 p resp.response

data/examples/vmware.rb

@@ -10,9 +10,19 @@ pool "poolparty" do
       image_id  "/Users/nmurray/Documents/VMware/Ubuntu-jaunty.vmwarevm/Ubuntu-jaunty.vmx"
       public_ip "192.168.133.128"
     end
+    has_hermes
     has_file "/etc/motd", :content => "welcome to your instance"
 
     has_convenience_helpers
   end
 
 end
+
+# hermes
+# want to be able to run cloud configure to update it
+# but want to be able to pull from locally to configure it
+# submodules?  
+# node name
+# install path
+# how to upload the escript target system
+# how to upload the tar.gz of the release

data/lib/cloud_providers/cloud_provider_instance.rb

@@ -66,16 +66,16 @@ module CloudProviders
         bootstrap! unless bootstrapped?
         callback :before_configure
         raise StandardError.new("You must pass in a cloud to configure an instance") unless cloud
-        cloud.compile(self)
+        cloud.compile(self)        
+        
         scp(:source       => keypair.full_filepath, 
             :destination  => "/etc/poolparty/keys/#{keypair.basename}")
-        script_file = Provision::Bootstrapper.configure_script(cloud, os)
         
         FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty" unless File.directory?(cloud.tmp_path/"etc"/"poolparty")
-        FileUtils.cp script_file, cloud.tmp_path/"etc"/"poolparty"
+        pack_clouds_dot_rb_and_expected_directories
         
+        dputs("Rsyncing #{cloud.tmp_path/"*"}")
         rsync(:source => cloud.tmp_path/"*", :destination => "/")
-        run("chmod +x /etc/poolparty/#{File.basename(script_file)}; /bin/sh /etc/poolparty/#{File.basename(script_file)}").chomp
         run(cloud.dependency_resolver.compile_command)
         callback :after_configure
       end
@@ -95,6 +95,15 @@ module CloudProviders
       end
       alias :platform :os  # Chef uses platform, aliased for conveneince
       
+      def pack_clouds_dot_rb_and_expected_directories
+        %w(lib plugins).each do |dir|
+          if File.directory?(d = cloud.clouds_dot_rb_dir/dir)
+            dputs("Adding local path: #{d}")
+            FileUtils.cp_r d, cloud.tmp_path/cloud.base_config_directory, :verbose => true, :remove_destination => true # req'd for symlinks
+          end
+        end
+        FileUtils.cp cloud.clouds_dot_rb_file, cloud.tmp_path/"/etc/poolparty/clouds.rb"
+      end
       # Determine the os
       # Default to ubuntu
       # Send the determine_os.sh script to the node and run it remotely

data/lib/cloud_providers/connections.rb

@@ -51,7 +51,7 @@ module  CloudProviders
       raise StandardError.new("You must pass a :source=>uri option to rsync") unless opts[:source]
       destination_path = opts[:destination] || opts[:source]
       rsync_opts = opts[:rsync_opts] || '-va'
-      cmd_string =  "rsync -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
+      cmd_string =  "rsync -L -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
       out = system_run(cmd_string)
       out
     end

data/lib/core/file.rb

@@ -0,0 +1,12 @@
+class File
+  class << self
+    alias_method :old_symlink, :symlink
+    def symlink(old_name, new_name)
+      begin
+        old_symlink(old_name, new_name)
+      rescue Errno::EEXIST
+        $stderr.puts "warning: symlinking #{old_name} -> #{new_name}. Already exists"
+      end
+    end
+  end
+end

data/lib/dependency_resolvers/base.rb

@@ -111,7 +111,7 @@ module DependencyResolvers
       when Symbol
         ":#{obj}"
       when Hash
-        "#{obj.map {|k,v| ":#{k} => #{handle_print_variable(v)}" unless v == obj }.compact.join(",\n")}"
+        "{#{obj.map {|k,v| ":#{k} => #{handle_print_variable(v)}" unless v == obj }.compact.join(",")}}"
       else
         "#{obj}"
       end

data/lib/dependency_resolvers/chef.rb

@@ -159,13 +159,15 @@ module DependencyResolvers
         file_pointers.each do |n,f|
           f << "\n#{n} Mash.new unless attribute?(\"#{n}\")\n"
         end
-        variables.each do |var|
-          
-          var_val = handle_print_variable(ProxyObject.new(var, @caller).compile(:value))
+        variables.each do |var|          
           if var.parent && !var.parent.is_a?(PoolParty::Cloud)
-            file_pointers[var.parent.has_method_name] << "#{var.parent.has_method_name}[:#{var.name}] = #{var_val}\n"
+            file_pointers[var.parent.has_method_name] << "#{var.parent.has_method_name}[:#{var.name}] ="
+            file_pointers[var.parent.has_method_name] << ProxyObject.new(var, @caller).compile(:value)
+            file_pointers[var.parent.has_method_name] << "\n"
           else
-            file_pointers[:poolparty] << "poolparty[:#{var.name}] = #{var_val}\n"
+            file_pointers[:poolparty] << "poolparty[:#{var.name}] = "
+            file_pointers[:poolparty] << ProxyObject.new(var, @caller).compile(:value)
+            file_pointers[:poolparty] << "\n"
           end
         end
         # Close the files

data/lib/dependency_resolvers/proxy_object.rb

@@ -32,15 +32,18 @@ module DependencyResolvers
         meth_name
       when Symbol
         @current_printing_method = meth_name
-        self.send(meth_name).to_s
+        self.send(meth_name)
       else
         raise PoolParty::PoolPartyError.create("ProxyObjectError", "Compilation of #{proxy.inspect} error. Strings and symbols are supported")
       end
+      
+      str = handle_print_variable(str) if proxy.class == PoolParty::Resources::Variable
+      
       begin
         ERB.new(str).result(self.send(:binding))
       rescue Exception => e
-        p [:error, e, str]
-      end      
+        ""
+      end
     end
     
     # Print the dsl options in the Erb string format
@@ -73,6 +76,11 @@ module DependencyResolvers
       end.join("\n")
     end
     
+    # Print variables
+    def handle_print_variable(var)
+      DependencyResolvers::Base.handle_print_variable(var)
+    end
+    
     def instance
       @caller
     end

data/lib/poolparty.rb

@@ -43,7 +43,9 @@ require "poolparty/pool_party_error"
     hash
     symbol
     proc
-    time).each do |lib|
+    time
+    file
+  ).each do |lib|
   require "core/#{lib}"
 end
 

data/lib/poolparty/base.rb

@@ -37,8 +37,7 @@ module PoolParty
     # - stack
     def run_in_context(o={}, &block)
       proc = Proc.new do
-        # Name MUST be set first
-        set_vars_from_options(:name => o[:name]) if o.has_key?(:name)
+        set_vars_from_options(:name => o[:name]) if o.has_key?(:name) # Name MUST be set first
         set_vars_from_options(o)
         instance_eval &block if block
       end

data/lib/poolparty/cloud.rb

@@ -27,6 +27,8 @@ module PoolParty
     end
     
     def before_compile
+      add_monitoring_stack_if_needed
+      
       validate_all_resources
     end
     
@@ -61,7 +63,7 @@ module PoolParty
     end
     
     # Cloud provider methods
-    def nodes(o={}); delayed_action {cloud_provider.nodes(o)}; end
+    def nodes(o={}); delayed_action {cloud_provider.nodes(o).collect{|n| n.cloud = self; n}}; end
     def run_instance(o={}); cloud_provider.run_instance(o);end
     def terminate_instance!(o={}); cloud_provider.terminate_instance!(o);end
     def describe_instances(o={}); cloud_provider.describe_instances(o);end
@@ -87,7 +89,7 @@ module PoolParty
     
     # 1.) Launches a new instance,
     # 2.) Waits for the instance to get an ip address
-    # 3.) Waits for port 22 to be open
+    # 3.) Waits for port ssh_port to be open
     # 4.) Calls call_after_launch_instance callbacks
     # 5.) Executes passed &block, if any
     # 6.) Returns the new instance object
@@ -98,7 +100,7 @@ module PoolParty
       instance.cloud = self
       @instance = instance
       #wait for an ip and then wait for ssh port, then configure instance
-      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(22, :timeout=>timeout)
+      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(ssh_port, :timeout=>timeout)
         callback :after_launch_instance
         instance.callback :before_bootstrap
         instance.bootstrap!
@@ -109,7 +111,7 @@ module PoolParty
         block.call(instance) if block
         instance
       else
-         raise StandardError.new("Instance port 22 not available")
+         raise StandardError.new("Instance port #{ssh_port} not available")
       end
       instance.refresh!
       instance
@@ -126,7 +128,7 @@ module PoolParty
     
     # convenience method to loop thru all the nodes and configure them
     def configure!(opts={}, threaded=true)
-      nodes.collect{|n| n.configure!}
+      nodes.collect{|n| n.configure! }
     end
     
     # Run command/s on all nodes in the cloud.
@@ -177,6 +179,19 @@ module PoolParty
       end
     end
     
+    # Add the monitoring stack
+    def add_monitoring_stack_if_needed
+      if monitors.size > 0
+        
+        run_in_context do
+          %w(collectd hermes).each do |m|
+            self.send m.to_sym
+          end
+        end
+        
+      end
+    end
+    
     # Take the cloud's resources and compile them down using 
     # the defined (or the default dependency_resolver, chef)
     def compile(caller=nil)

data/lib/poolparty/default.rb

@@ -11,6 +11,7 @@ module PoolParty
     default_options(
       :user => "root",
       :os => :ubuntu,
+      :ssh_port => 22,
       :tmp_path => "/tmp/poolparty",
       :poolparty_home_path => "#{ENV["HOME"]}/.poolparty",
       :ec2_home => "#{ENV["HOME"]}/.ec2",

data/lib/poolparty/plugin.rb

@@ -2,11 +2,7 @@
   Simply a stub class for documentation purposes
   Plugins are all resources
 =end
-module PoolParty
-  class Plugin
-  end
-end
 
-%w(apache git rails).each do |plugin|
+%w(apache git rails collectd hermes).each do |plugin|
   require "plugins/#{plugin}"
 end

data/lib/poolparty/plugins/apache.rb

@@ -50,7 +50,7 @@ module PoolParty
           passenger_configs
           
           has_exec "install_passenger_script" do
-            command "passenger-install-apache2-module -auto"
+            command "passenger-install-apache2-module --auto"
             notifies get_exec("restart-apache2"), :run
             requires get_exec("restart-apache2")
             requires get_package("apache2")
@@ -157,14 +157,16 @@ PassengerRuby <%= @node[:languages][:ruby][:ruby_bin] %>
       end
       
       def install_site(name, opts={})
-        opts.merge!(:name => "/etc/apache2/sites-available/#{name}")
+        sitename = name
+
+        opts.merge!(:name => "/etc/apache2/sites-available/#{sitename}")
         has_directory(:name => "/etc/apache2/sites-available")
         has_file(opts) unless opts[:no_file]
-        has_exec(:name => "/usr/sbin/a2ensite #{name}") do
+        has_exec(:name => "/usr/sbin/a2ensite #{sitename}") do
           notifies get_exec("reload-apache2"), :run
           requires get_exec("reload-apache2")
-          requires get_file("/etc/apache2/sites-available/#{name}")
-          not_if "/bin/sh -c '[ -L /etc/apache2/sites-enabled/#{name} ] && [ /etc/apache2/sites-enabled/#{name} -ef /etc/apache2/sites-available/#{name} ]'"
+          requires get_file("/etc/apache2/sites-available/#{sitename}")
+          not_if "/bin/sh -c '[ -L /etc/apache2/sites-enabled/#{sitename} ] && [ /etc/apache2/sites-enabled/#{sitename} -ef /etc/apache2/sites-available/#{sitename} ]'"
         end
       end
       
@@ -202,4 +204,4 @@ end
 $:.unshift(File.dirname(__FILE__))
 %w(php5 virtual_host passenger_site).each do |lib|
   require "apache2/#{lib}"
-end
+end

data/lib/poolparty/plugins/collectd.rb

@@ -0,0 +1,24 @@
+module PoolParty
+  module Resources
+  
+    class Collectd < Resource
+      
+      PoolParty::Resources::FileResource.has_searchable_paths(:prepend_paths=> [File.dirname(__FILE__)+'/collectd/templates'])
+      
+      def after_loaded
+        has_package 'collectd'
+        %w(rrdtool librrd-dev librrd-ruby  libsensors-dev libsnmp-dev collectd collectd-dev).each{|pkg| 
+          has_package pkg 
+        }
+        has_gem_package "astro-collectd"
+        
+        has_variable 'server', "localhost"
+        has_file '/etc/collectd/collectd.conf' do
+          template 'collectd.conf.erb'
+        end
+      end
+      
+    end
+  
+  end
+end