poolparty 1.3.4 → 1.3.6

data/Rakefile

@@ -39,8 +39,8 @@ end
 
 # Generate documentation
 Rake::RDocTask.new do |rd|
-  rd.main = "Readme.txt"
-  rd.rdoc_files.include("Readme.txt", "lib/**/*.rb")
+  rd.main = "README.rdoc"
+  rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
   rd.rdoc_dir = "rdoc"
   # rd.template = "hanaa"
 end

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :minor: 3
-:patch: 4
+:patch: 6
 :major: 1

data/bin/cloud-bootstrap

@@ -21,6 +21,7 @@ EOS
     @loaded_clouds.each do |cld|
       
       cld.nodes.each do |node|
+        node.cloud = cld # needed for bootstrap
         node.bootstrap!(:force=>command[:force])
       end
       

data/bin/cloud-configure

@@ -26,6 +26,7 @@ EOS
       end
       
       cld.nodes.each do |node|
+        node.cloud = cld # ?
         node.configure!(:cloud => cld)
       end
       

data/bin/cloud-contract

@@ -17,6 +17,7 @@ EOS
     @loaded_clouds.each do |cld|
       
       node_to_terminate = cld.nodes.last
+      node_to_terminate.cloud = cld
       
       msg = [
         "Contracting cloud #{cld.name} (#{cld.keypair}) by 1",

data/bin/cloud-misc

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+$:.unshift(File.dirname(__FILE__) + "/../lib")
+require "poolparty"
+
+require 'git-style-binary/command'
+
+GitStyleBinary.command do
+  banner <<-EOS
+Usage: #{$0} #{all_options_string}
+
+  read the code 
+EOS
+
+  short_desc "various cloud development tasks"
+  
+  opt :inst_num, "The number of the instance to run bootstrap on", :type => :int
+  opt :force, "Force rebootstrapping (useful for upgrading)", :default => false
+
+  run do |command|
+    @loaded_clouds.each do |cld|
+      cld.nodes.each do |node|
+
+        case command.argv.shift
+        when "deploy_hermes"
+          hermesdir = File.dirname(__FILE__) + "/../vendor/erlang/hermes"
+          release = `#{hermesdir}/scripts/most_recent_release tar.gz`.strip
+          node.scp(:source => release, :destination => "/var/poolparty/tmp")
+          node.scp(:source => "#{hermesdir}/scripts/target_system", :destination => "/var/poolparty/tmp")
+        end
+
+      end
+    end
+  end
+end

data/bin/cloud-setup

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+$:.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+require "poolparty"
+require "poolparty/installer"
+
+require 'git-style-binary/command'
+
+GitStyleBinary.command do
+  banner <<-EOS
+Usage: #{$0} #{all_options_string}
+
+  Sets up a basic clouds.rb file and carries through the steps to bootstrap a new cloud
+EOS
+
+  short_desc "Setup a new clouds.rb file"
+  
+  providers = {}
+  PoolParty::Installer.all.each_with_index do |installer,idx|
+    providers.merge!(idx+1 => installer)
+  end
+  
+  str = ["The cloud_provider you'd like to set your clouds.rb for. The available clouds include:"]
+  PoolParty::Installer.all.each do |provider|
+    str << "\t\t\t\t\t\t\t\t#{provider.name} - #{provider.description}"
+  end
+  opt :type, str.join("\n"), :type => :string, :default => "ec2"
+
+  run do |command|
+    
+    klass = PoolParty::Installer.find_by_name(command[:type])
+    puts "Great, we'll be using #{klass}"
+    inst = klass.send :new
+    inst.run
+    
+  end
+end

data/bin/cloud-ssh

@@ -13,11 +13,14 @@ EOS
 
   short_desc "ssh into an instance of the cloud"
   
+  opt :instance_number, "Ssh into the instance number", :type => :integer, :default => 0
+  
   run do |command|
     cloud = @loaded_clouds.first
     
     if !cloud.nodes.empty?
-      cloud.nodes.first.ssh
+      inst = cloud.nodes[command[:instance_number]]
+      inst ? inst.ssh : puts("Error: The instance number (#{command[:instance_number]}) is too high for the current number of instances")
     else
       puts "No running instances can be found"
     end

data/config/jeweler.rb

@@ -26,11 +26,12 @@ end
 
     s.files = (%w(Rakefile README.rdoc License.txt VERSION.yml) + Dir["{config,examples,lib,test,tasks,script,generators,bin,vendor}/**/*"])
     
-    s.files.exclude '**/*.beam'
-    s.files.exclude "**/*/erl_crash.dump"
+    s.files.exclude 'vendor/erlang/*'
+    s.files.exclude 'examples/thrift/**/*.beam'
+    # s.files.exclude "**/*/erl_crash.dump"
     
     s.has_rdoc = true
-    s.extra_rdoc_files = ["README.txt", "License.txt", 'History.txt']
+    s.extra_rdoc_files = ["README.rdoc", "License.txt", 'History.txt']
     s.require_paths = ["lib"]
     s.rdoc_options = ['--quiet', '--title', 'PoolParty documentation',
         # "index.html",

data/examples/monitored_cloud.rb

@@ -10,7 +10,7 @@ pool "poolparty" do
       image_id 'emi-39CA160F'
     end
     
-    monitor :cpu do |c|
+    monitor :'cpu-idle' do |c|
       vote_for(:expand) if c > 0.9
       vote_for(:contract) if c < 0.1
     end

data/examples/thrift/thrift_example.rb

@@ -3,7 +3,7 @@
 $:.unshift(File.dirname(__FILE__) + "/../../lib")
 $:.unshift(File.dirname(__FILE__) + "/../../examples")
 require "poolparty"
-require "simple"
+# require "simple"
 
 $:.unshift("#{File.dirname(__FILE__)}/../../lib/proto/gen-rb")
 
@@ -15,6 +15,7 @@ require "poolparty_types"
 port = ARGV.pop || 11223
 
 transport = Thrift::BufferedTransport.new(Thrift::Socket.new('localhost', port))
+# transport = Thrift::BufferedTransport.new(Thrift::Socket.new('vm', port))
 protocol = Thrift::BinaryProtocol.new(transport)
 
 client = CloudThrift::CommandInterface::Client.new(protocol)
@@ -22,6 +23,7 @@ transport.open()
 
 cld = CloudThrift::CloudQuery.new
 cld.name = 'monitored_app'
+# cld.name = 'vmware'
 
 resp = client.run_command(cld, "name", [])
 puts resp.response
@@ -29,8 +31,8 @@ puts resp.response
 resp = client.run_command(cld, "maximum_instances", [])
 puts resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.91"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.91"])
 p resp.response
 
-resp = client.run_command(cld, "run_monitor", ["cpu", "0.01"])
+resp = client.run_command(cld, "run_monitor", ["cpu-idle", "0.01"])
 p resp.response

data/examples/vmware.rb

@@ -0,0 +1,28 @@
+require "poolparty"
+require 'poolparty-extensions'
+
+pool "poolparty" do
+  instances 1
+  
+  cloud "vmware" do
+    keypair "id_rsa"
+    using :vmware do
+      image_id  "/Users/nmurray/Documents/VMware/Ubuntu-jaunty.vmwarevm/Ubuntu-jaunty.vmx"
+      public_ip "192.168.133.128"
+    end
+    has_hermes
+    has_file "/etc/motd", :content => "welcome to your instance"
+
+    has_convenience_helpers
+  end
+
+end
+
+# hermes
+# want to be able to run cloud configure to update it
+# but want to be able to pull from locally to configure it
+# submodules?  
+# node name
+# install path
+# how to upload the escript target system
+# how to upload the tar.gz of the release

data/lib/cloud_providers/cloud_provider_instance.rb

@@ -66,16 +66,16 @@ module CloudProviders
         bootstrap! unless bootstrapped?
         callback :before_configure
         raise StandardError.new("You must pass in a cloud to configure an instance") unless cloud
-        cloud.compile(self)
+        cloud.compile(self)        
+        
         scp(:source       => keypair.full_filepath, 
             :destination  => "/etc/poolparty/keys/#{keypair.basename}")
-        script_file = Provision::Bootstrapper.configure_script(cloud, os)
         
         FileUtils.mkdir_p cloud.tmp_path/"etc"/"poolparty" unless File.directory?(cloud.tmp_path/"etc"/"poolparty")
-        FileUtils.cp script_file, cloud.tmp_path/"etc"/"poolparty"
+        pack_clouds_dot_rb_and_expected_directories
         
+        dputs("Rsyncing #{cloud.tmp_path/"*"}")
         rsync(:source => cloud.tmp_path/"*", :destination => "/")
-        run("chmod +x /etc/poolparty/#{File.basename(script_file)}; /bin/sh /etc/poolparty/#{File.basename(script_file)}").chomp
         run(cloud.dependency_resolver.compile_command)
         callback :after_configure
       end
@@ -95,6 +95,15 @@ module CloudProviders
       end
       alias :platform :os  # Chef uses platform, aliased for conveneince
       
+      def pack_clouds_dot_rb_and_expected_directories
+        %w(lib plugins).each do |dir|
+          if File.directory?(d = cloud.clouds_dot_rb_dir/dir)
+            dputs("Adding local path: #{d}")
+            FileUtils.cp_r d, cloud.tmp_path/cloud.base_config_directory, :verbose => true, :remove_destination => true # req'd for symlinks
+          end
+        end
+        FileUtils.cp cloud.clouds_dot_rb_file, cloud.tmp_path/"/etc/poolparty/clouds.rb"
+      end
       # Determine the os
       # Default to ubuntu
       # Send the determine_os.sh script to the node and run it remotely
@@ -222,7 +231,7 @@ module CloudProviders
       end
             
       def on_all_callbacks(call_time, *args, &block)
-        cloud.callback call_time
+        cloud.callback call_time if cloud
         super
       end
       

data/lib/cloud_providers/connections.rb

@@ -51,7 +51,7 @@ module  CloudProviders
       raise StandardError.new("You must pass a :source=>uri option to rsync") unless opts[:source]
       destination_path = opts[:destination] || opts[:source]
       rsync_opts = opts[:rsync_opts] || '-va'
-      cmd_string =  "rsync -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
+      cmd_string =  "rsync -L -e 'ssh #{ssh_options}' #{rsync_opts} #{opts[:source]}  #{user}@#{host}:#{destination_path}"
       out = system_run(cmd_string)
       out
     end

data/lib/core/file.rb

@@ -0,0 +1,12 @@
+class File
+  class << self
+    alias_method :old_symlink, :symlink
+    def symlink(old_name, new_name)
+      begin
+        old_symlink(old_name, new_name)
+      rescue Errno::EEXIST
+        $stderr.puts "warning: symlinking #{old_name} -> #{new_name}. Already exists"
+      end
+    end
+  end
+end

data/lib/core/object.rb

@@ -71,10 +71,10 @@ class Object
     puts "[INFO] -- #{m}" if verbose?
   end
   def dputs(m="")
-    puts "[DEBUG] -- #{m.inspect}" if debugging?
+    puts "[DEBUG] -- #{m.is_a?(String) ? m : m.inspect}" if debugging?
   end
   def ddputs(m="")
-    puts "[VERY DEBUG] -- #{m.inspect}" if very_debugging?
+    puts "[VERY DEBUG] -- #{m.is_a?(String) ? m : m.inspect}" if very_debugging?
   end
   def verbose?
     $TESTING ||= false

data/lib/dependency_resolvers/base.rb

@@ -111,7 +111,7 @@ module DependencyResolvers
       when Symbol
         ":#{obj}"
       when Hash
-        "#{obj.map {|k,v| ":#{k} => #{handle_print_variable(v)}" unless v == obj }.compact.join(",\n")}"
+        "{#{obj.map {|k,v| ":#{k} => #{handle_print_variable(v)}" unless v == obj }.compact.join(",")}}"
       else
         "#{obj}"
       end

data/lib/dependency_resolvers/chef.rb

@@ -26,7 +26,7 @@ module DependencyResolvers
       end
       
       def compile_command
-        "/usr/bin/chef-solo -c /etc/chef/solo.rb -j /etc/chef/dna.json"
+        "/usr/bin/chef-solo -c /etc/chef/solo.rb -j /etc/chef/dna.json#{ " -l debug" if very_debugging?}"
       end
          
       # compile the resources
@@ -57,7 +57,7 @@ module DependencyResolvers
           super
         end
         
-        apply_meta_functions(res, o) if res.is_a?(PoolParty::Resource)
+        apply_meta_functions(res, o) if res.is_a?(PoolParty::Resource) && res.print_to_chef != :no_print
       end
       
       default_attr_reader :variables, []
@@ -159,13 +159,15 @@ module DependencyResolvers
         file_pointers.each do |n,f|
           f << "\n#{n} Mash.new unless attribute?(\"#{n}\")\n"
         end
-        variables.each do |var|
-          
-          var_val = handle_print_variable(ProxyObject.new(var, @caller).compile(:value))
+        variables.each do |var|          
           if var.parent && !var.parent.is_a?(PoolParty::Cloud)
-            file_pointers[var.parent.has_method_name] << "#{var.parent.has_method_name}[:#{var.name}] = #{var_val}\n"
+            file_pointers[var.parent.has_method_name] << "#{var.parent.has_method_name}[:#{var.name}] ="
+            file_pointers[var.parent.has_method_name] << ProxyObject.new(var, @caller).compile(:value)
+            file_pointers[var.parent.has_method_name] << "\n"
           else
-            file_pointers[:poolparty] << "poolparty[:#{var.name}] = #{var_val}\n"
+            file_pointers[:poolparty] << "poolparty[:#{var.name}] = "
+            file_pointers[:poolparty] << ProxyObject.new(var, @caller).compile(:value)
+            file_pointers[:poolparty] << "\n"
           end
         end
         # Close the files

data/lib/dependency_resolvers/proxy_object.rb

@@ -32,15 +32,18 @@ module DependencyResolvers
         meth_name
       when Symbol
         @current_printing_method = meth_name
-        self.send(meth_name).to_s
+        self.send(meth_name)
       else
         raise PoolParty::PoolPartyError.create("ProxyObjectError", "Compilation of #{proxy.inspect} error. Strings and symbols are supported")
       end
+      
+      str = handle_print_variable(str) if proxy.class == PoolParty::Resources::Variable
+      
       begin
         ERB.new(str).result(self.send(:binding))
       rescue Exception => e
-        p [:error, e, str]
-      end      
+        ""
+      end
     end
     
     # Print the dsl options in the Erb string format
@@ -73,6 +76,11 @@ module DependencyResolvers
       end.join("\n")
     end
     
+    # Print variables
+    def handle_print_variable(var)
+      DependencyResolvers::Base.handle_print_variable(var)
+    end
+    
     def instance
       @caller
     end

data/lib/mixins/askable.rb

@@ -15,6 +15,7 @@ module Askable
         colored_say e.inspect
         exit 0
       end
+      answer
     end
 
     def ask_with_help(opts={}, &block)
@@ -56,13 +57,9 @@ module Askable
     
     def choose(str, choices={}, opts={}, &block)
       colored_say(str)
-      choices.each do |k,v|
-        colored_say("#{k}) #{v}")
-      end
-      colored_print((opts[:prompt] || "> "))
-      pick = gets.chomp.to_i
-      yield choices[pick] if block
-      choices[pick]
+      answer = ask(choices)
+      yield answer if block
+      answer
     end
     
     def wait
@@ -130,6 +127,8 @@ module Askable
       case @question
       when String
         ask_string
+      when Hash
+        ask_hash
       when Array
         ask_array
       end
@@ -150,5 +149,15 @@ module Askable
       num = (@input.gets).to_i rescue colored_say("<red>Invalid input, must be a number between 1 and #{@question.size + 1}")
       @answer = @question[num-1]
     end
+    
+    def ask_hash
+      @question.sort.each do |k,v|
+        colored_say("#{k}) #{v}")
+      end
+      colored_print((@opts[:prompt] || "> "))
+      pick = @input.gets.to_i
+      @answer = @question[pick]
+    end
+    
   end
 end

data/lib/poolparty/base.rb

@@ -37,6 +37,7 @@ module PoolParty
     # - stack
     def run_in_context(o={}, &block)
       proc = Proc.new do
+        set_vars_from_options(:name => o[:name]) if o.has_key?(:name) # Name MUST be set first
         set_vars_from_options(o)
         instance_eval &block if block
       end
@@ -127,12 +128,12 @@ module PoolParty
     # to create edges on the graph
     def resources_graph(force=false)
       return @resources_graph if @resources_graph && !force
-      result = Digraph.new
-      
-      add_ordered_resources_to_result(resources, result)
+      result = Digraph.new      
       
       create_graph(resources, nil, result)
       
+      add_ordered_resources_to_result(resources, result)
+      
       @resources_graph = result
     end
     
@@ -141,7 +142,7 @@ module PoolParty
       arr_of_resources = resources.zip_offset(1)
       
       arr_of_resources.each do |first, second|
-        result.add_edge!(first, second)
+        result.add_edge!(first, second) unless result.edge?(first, second) or result.edge?(second, first)
         add_ordered_resources_to_result(first.resources, result)
       end
     end
@@ -200,9 +201,9 @@ module PoolParty
     
     # The clouds.rb file
     def clouds_dot_rb_file(n=nil); self.class.clouds_dot_rb_file(n); end
-    def self.clouds_dot_rb_file(n=nil)
-        Pool.clouds_dot_rb_file(n)
-    end
+    def self.clouds_dot_rb_file(n=nil);Pool.clouds_dot_rb_file(n);end
+    def clouds_dot_rb_dir(n=nil); self.class.clouds_dot_rb_dir(n);end
+    def self.clouds_dot_rb_dir(n=nil); Pool.clouds_dot_rb_dir(n);end
     
     # If the method is missing from ourself, check the Default
     # class for the corresponding method.

data/lib/poolparty/cloud.rb

@@ -26,6 +26,12 @@ module PoolParty
     callback_block do |cld, callback|
     end
     
+    def before_compile
+      add_monitoring_stack_if_needed
+      
+      validate_all_resources
+    end
+    
     # Freeze the cloud_name so we can't modify it at all, set the plugin_directory
     # call and run instance_eval on the block and then call the after_create callback
     def initialize(n, o={}, &block)
@@ -57,7 +63,7 @@ module PoolParty
     end
     
     # Cloud provider methods
-    def nodes(o={}); delayed_action {cloud_provider.nodes(o)}; end
+    def nodes(o={}); delayed_action {cloud_provider.nodes(o).collect{|n| n.cloud = self; n}}; end
     def run_instance(o={}); cloud_provider.run_instance(o);end
     def terminate_instance!(o={}); cloud_provider.terminate_instance!(o);end
     def describe_instances(o={}); cloud_provider.describe_instances(o);end
@@ -83,7 +89,7 @@ module PoolParty
     
     # 1.) Launches a new instance,
     # 2.) Waits for the instance to get an ip address
-    # 3.) Waits for port 22 to be open
+    # 3.) Waits for port ssh_port to be open
     # 4.) Calls call_after_launch_instance callbacks
     # 5.) Executes passed &block, if any
     # 6.) Returns the new instance object
@@ -94,7 +100,7 @@ module PoolParty
       instance.cloud = self
       @instance = instance
       #wait for an ip and then wait for ssh port, then configure instance
-      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(22, :timeout=>timeout)
+      if instance.wait_for_public_ip(timeout) && instance.wait_for_port(ssh_port, :timeout=>timeout)
         callback :after_launch_instance
         instance.callback :before_bootstrap
         instance.bootstrap!
@@ -105,7 +111,7 @@ module PoolParty
         block.call(instance) if block
         instance
       else
-         raise StandardError.new("Instance port 22 not available")
+         raise StandardError.new("Instance port #{ssh_port} not available")
       end
       instance.refresh!
       instance
@@ -122,7 +128,7 @@ module PoolParty
     
     # convenience method to loop thru all the nodes and configure them
     def configure!(opts={}, threaded=true)
-      nodes.collect{|n| n.configure!}
+      nodes.collect{|n| n.configure! }
     end
     
     # Run command/s on all nodes in the cloud.
@@ -173,12 +179,31 @@ module PoolParty
       end
     end
     
+    # Add the monitoring stack
+    def add_monitoring_stack_if_needed
+      if monitors.size > 0
+        
+        run_in_context do
+          %w(collectd hermes).each do |m|
+            self.send m.to_sym
+          end
+        end
+        
+      end
+    end
+    
     # Take the cloud's resources and compile them down using 
     # the defined (or the default dependency_resolver, chef)
     def compile(caller=nil)
+      callback :before_compile
       FileUtils.mkdir_p tmp_path unless File.directory?(tmp_path)
-      ddputs("Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"}")
-      dependency_resolver.compile_to(ordered_resources, tmp_path/"etc"/"#{dependency_resolver_name}", caller)
+      ddputs <<-EOE
+Compiling cloud #{self.name} to #{tmp_path/"etc"/"#{dependency_resolver_name}"} 
+  number of resources: #{ordered_resources.size}
+      EOE
+      out = dependency_resolver.compile_to(ordered_resources, tmp_path/"etc"/"#{dependency_resolver_name}", caller)
+      callback :after_compile
+      out
     end
     
     # Get the os of the first node if it was not explicity defined, we'll assume they are
@@ -242,5 +267,50 @@ module PoolParty
       end
     end
     
+    def validate_all_resources
+      ddputs("Validating all the resources")
+      [:ensure_not_cyclic, :ensure_meta_fun_are_resources].each do |meth|
+        self.send meth
+      end
+    end
+    
+    def ensure_not_cyclic
+      if resources_graph.cyclic?
+        cycles = []
+        
+        resources_graph.edges.each do |edge|
+          if resources_graph.adjacent?(edge.source, edge.target) && resources_graph.adjacent?(edge.target, edge.source) && !cycles.include?(edge.source)
+            cycles << "#{edge.source.class}(#{edge.source.name}) depends on #{edge.target.class}(#{edge.target.name})"
+          end
+        end
+        msg =<<-EOE
+      
+        Your resource graph is cyclic. Two resources depend on each other, Cannot decide which resource
+        to go first. Dying instead. Correct this and then try again.
+      
+          #{cycles.join("\n          ")}
+      
+        Hint: You can see the resource graph by generating it with:
+          cloud compile -g name
+        
+        EOE
+        raise PoolPartyError.create("CyclicResourceGraphError", msg)
+      end
+    end
+    
+    def ensure_meta_fun_are_resources
+      resources.each do |res|
+        
+        if res.meta_notifies
+          res.meta_notifies.each do |ty, arr|
+            arr.each do |nm, action|
+              raise PoolPartyError.create("ResourceNotFound", "A resource required for #{ty}(#{nm}) was not found: #{ty}(#{nm}). Please make sure you've specified this in your configuration.") unless get_resource(ty, nm)
+            end
+          end
+        end
+        
+      end
+    end    
+    
   end
 end

data/lib/poolparty/default.rb

@@ -11,6 +11,7 @@ module PoolParty
     default_options(
       :user => "root",
       :os => :ubuntu,
+      :ssh_port => 22,
       :tmp_path => "/tmp/poolparty",
       :poolparty_home_path => "#{ENV["HOME"]}/.poolparty",
       :ec2_home => "#{ENV["HOME"]}/.ec2",

data/lib/poolparty/installer.rb

@@ -15,7 +15,7 @@ module PoolParty
         steps.each {|c| self.__send__ c.to_sym }
         self.__send__ :closing_message
       rescue Exception => e
-        # colored_say e.inspect
+        colored_say e.inspect
         colored_say exit_msg
       ensure
         Colors.reset!
@@ -35,7 +35,7 @@ First, we'll setup your environment so using PoolParty will be a <blue>breeze</b
 <yellow>Cancelled PoolParty installation</yellow>
 
 You can always restart this by typing:
-<blue>install-poolparty</blue>
+<blue>cloud setup</blue>
 
 <line>
 EOE
@@ -97,8 +97,12 @@ irc.freenode.net / #poolpartyrb
     def self.all
       @all ||= []
     end
-
-
+    
+    def self.find_by_name(nm)
+      all.detect do |i|
+        i.name =~ /#{nm}/i
+      end
+    end
   end
 end