auser-poolparty-extensions 0.0.7 → 0.1.1

data/README.rdoc

@@ -10,6 +10,21 @@
 
 = Available extensions
 
+== bash_alias
+  Adds a bash alias
+
+== development_gem_package
+  Deploy a and install a gem developed locally. This is useful when you are
+  developing an internal or forked gem and you want to deploy it to your cloud
+
+
+  == Usage
+
+    has_development_gem_package('jetty-rack', 
+                                :from => "~/path/to/my/site", 
+                                :git_pull_first => true)  # git pull from before sending to server
+
+
 == dynomite
   Installs edge dynomite from the git repository
   

data/Rakefile

@@ -8,7 +8,7 @@ begin
     gem.summary = %Q{Extensions on to of poolparty}
     gem.email = "arilerner@mac.com"
     gem.homepage = "http://github.com/auser/poolparty-extensions"
-    gem.authors = ["Ari Lerner"]
+    gem.authors = ["Ari Lerner, Nate Murray, & Michael Fairchild"]
 
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
@@ -76,7 +76,7 @@ task :readme do
   footer =<<-EOF
 == Copyright
 
-Copyright (c) 2009 Ari Lerner. See LICENSE for details.  
+Copyright (c) 2009 Ari Lerner & Cloudteam. See LICENSE for details.  
   EOF
   
   extensions = FileList["#{File.dirname(__FILE__)}/lib/extensions/*.rb"]

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
 :major: 0
-:minor: 0
-:patch: 7
+:minor: 1
+:patch: 1

data/lib/extensions/bash_alias.rb

@@ -1,3 +1,6 @@
+=begin rdoc
+  Adds a bash alias
+=end
 module PoolParty
   module Plugin
     class BashAlias < Plugin

data/lib/extensions/convenience_helpers.rb

@@ -0,0 +1,89 @@
+=begin rdoc
+
+== Convenience Helpers
+
+Adds a few handy packages and aliases for developing poolparty instances.
+
+== Usage
+
+    has_convenience_helpers
+
+== Examples
+
+    $ inspect-poolparty-recipes # => will vim the poolparty chef recipe file
+    $ cd-cookbooks              # => cd into the poolparty cookbooks directory
+    $ tree                      # => show the directory structure as a nice tree
+    /var/poolparty/dr_configure/chef/cookbooks/poolparty# tree
+    .
+    |-- attributes
+    |   `-- poolparty.rb
+    |-- recipes
+    |   `-- default.rb
+    `-- templates
+        `-- default
+            |-- etc
+            |   |-- jvm.erb
+            |   `-- motd.erb
+            |-- home
+            |   `-- hadoop
+            |       `-- ssh
+            |           `-- hadoop_id_rsa.erb
+            `-- usr
+                `-- local
+                    `-- hadoop
+                        `-- conf
+                            |-- hadoop-env.sh.erb
+                            `-- hadoop-site.xml.erb
+
+=end
+
+module PoolParty
+  module Plugin
+    class ConvenienceHelpers < Plugin
+      def before_load(o={}, &block)
+        add_packages
+        add_aliases
+        add_binaries
+        add_profile_updates
+      end
+
+      def add_packages
+        has_package "curl"
+        has_package "tree"
+        has_package "screen"
+
+        case cloud.os
+        when "centos"
+          has_package "ruby-irb"
+        else
+          has_package "irb"
+          has_package "vim-nox"
+        end
+      end
+
+      def add_aliases
+        has_bash_alias :name => "inspect-poolparty-recipes", :value => "vi /var/poolparty/dr_configure/chef/cookbooks/poolparty/recipes/default.rb"
+        has_bash_alias :name => "cd-cookbooks", :value => "pushd /var/poolparty/dr_configure/chef/cookbooks/poolparty"
+
+        %w{instance-id local-hostname local-ipv4 public-hostname public-ipv4 security-groups}.each do |metaalias|
+          has_bash_alias :name => metaalias, :value => "curl http://169.254.169.254/latest/meta-data/#{metaalias}"
+        end
+      end
+
+      def add_binaries
+        has_exec "wget http://gist.github.com/raw/131294/0622454b2cc2f787c04d20ab3d47e888e31edcd4/gistfile1 -O /usr/bin/xtail && chmod +x /usr/bin/xtail", 
+          :not_if => "test -e /usr/bin/xtail"
+        has_exec "curl http://timkay.com/aws/aws -o /usr/bin/aws", :not_if => "test -e /usr/bin/aws"
+      end
+
+      def add_profile_updates
+        has_exec %Q{echo \\"export PS1='\\\\u@\\\\h \\\\A \\\\w (#{cloud_name}) $ '\\" >> /root/.profile}, :not_if => "grep PS1 /root/.profile | grep #{cloud_name}"
+        # bind '"\e[A":history-search-backward'
+        # bind '"\e[B":history-search-forward'
+      end
+
+    end
+  end
+end
+
+

data/lib/extensions/development_gem_package.rb

@@ -1,5 +1,4 @@
-module PoolParty
-=begin  rdoc    
+=begin rdoc
 == Development Gem
 
 Deploy a and install a gem developed locally. This is useful when you are
@@ -13,6 +12,8 @@ developing an internal or forked gem and you want to deploy it to your cloud
                               :git_pull_first => true)  # git pull from before sending to server
 
 =end
+
+module PoolParty
   module Plugin
     class DevelopmentGem < Plugin
  

data/lib/extensions/ganglia/ganglia.rb

@@ -0,0 +1,219 @@
+=begin rdoc
+
+== Overview
+Install Ganglia cloud monitoring system
+
+== Requirements
+You'll need apache and php enabled in your clouds.rb. For example:
+
+    apache do
+      enable_php5 do
+        extras :gd
+      end
+    end
+
+Because the configs need to know about every node in the cloud *after* it has
+launched, you must setup an after_all_loaded block in your clouds.rb that calls
+ganglia.perform_after_all_loaded_for_master. For example:
+
+  after_all_loaded do
+    clouds[:hadoop_master].run_in_context do
+      ganglia.perform_after_all_loaded_for_master
+    end
+  end
+
+Currently the tasks only need to be run for master, so simply call this on your
+"master" cloud. Note: replace hadoop_master with the name of your cloud above.
+
+== EC2 Firewall
+
+ec2-authorize -P udp -p 8649 nmurray-hadoop
+ec2-authorize -P tcp -p 8649 nmurray-hadoop
+
+== References
+* http://www.ibm.com/developerworks/wikis/display/WikiPtype/ganglia?decorator=printable
+* http://docs.google.com/Doc?id=dgmmft5s_45hr7hmggr
+* http://www.hps.com/~tpg/notebook/ganglia.php
+* http://www.cultofgary.com/2008/10/16/ec2-and-ganglia/
+=end
+
+module PoolParty
+  module Plugin
+    class Ganglia < Plugin
+      def before_load(o={}, &block)
+        @monitored_features ||= {}
+        do_once do
+          install_dependencies
+          download
+        end
+      end
+
+      def install_dependencies
+        has_package :name => "rrdtool"
+        has_package :name => "build-essential"
+        has_package :name => "librrd-dev" 
+        has_package :name => "libapr1-dev"
+        has_package :name => "libconfuse-dev"
+        has_package :name => "libexpat1-dev"
+        has_package :name => "python-dev"
+
+        has_group "ganglia", :action => :create
+        has_user "ganglia", :gid => "ganglia"
+ 
+        # libart-2.0-2 ?
+      end
+
+      def download
+        has_exec "wget http://superb-west.dl.sourceforge.net/sourceforge/ganglia/ganglia-3.1.2.tar.gz -O /usr/local/src/ganglia-3.1.2.tar.gz",
+          :not_if => "test -e /usr/local/src/ganglia-3.1.2.tar.gz"
+        has_exec "cd /usr/local/src && tar -xvvf /usr/local/src/ganglia-3.1.2.tar.gz",
+          :not_if => "test -e /usr/local/src/ganglia-3.1.2"
+      end
+
+      def master
+        has_exec "cd /usr/local/src/ganglia-3.1.2 && ./configure --with-gmetad && make && make install",
+          :not_if => "test -e /usr/lib/ganglia"
+        has_exec "mv /usr/local/src/ganglia-3.1.2/web /var/www/ganglia",
+          :not_if => "test -e /var/www/ganglia"
+        has_file :name => "/var/www/ganglia/conf.php", :mode => "0644", :template => "ganglia-web-conf.php.erb"
+        has_variable "ganglia_gmond_is_master", :value => true
+        gmond
+        gmetad
+      end
+
+      def slave
+        has_exec "cd /usr/local/src/ganglia-3.1.2 && ./configure && make && make install",
+          :not_if => "test -e /usr/lib/ganglia"
+        has_variable "ganglia_gmond_is_master", :value => false
+        gmond
+      end
+
+      def gmond
+        has_directory "/etc/ganglia"
+        has_exec({:name => "restart-gmond", :command => "/etc/init.d/gmond restart", :action => :nothing})
+
+        has_file(:name => "/etc/init.d/gmond") do
+          mode 0755
+          template :bin/"gmond.erb"
+          notifies get_exec("restart-gmond"), :run
+        end
+
+        install_extra_gmond_monitors
+      end
+
+      def gmetad
+        has_directory "/var/lib/ganglia/rrds"
+        has_exec "chmod 755 /var/lib/ganglia/rrds"
+        has_exec "chown -R ganglia:ganglia /var/lib/ganglia/rrds"
+        has_exec({:name => "restart-gmetad", :command => "/etc/init.d/gmetad restart", :action => :nothing})
+        has_file(:name => "/etc/init.d/gmetad") do
+          mode 0755
+          template :bin/"gmetad.erb"
+          notifies get_exec("restart-gmetad"), :run
+        end
+      end
+
+      def monitor(*cloud_names)
+        @monitored_clouds = cloud_names
+      end
+
+      def perform_after_all_loaded_for_slave
+        gmond_after_all_loaded
+      end
+
+      def perform_after_all_loaded_for_master
+        raise "No clouds to monitor with ganglia specified. Please use the 'monitor(*cloud_names)' directive within your ganglia block" unless @monitored_clouds
+        gmond_after_all_loaded
+
+        data_sources = ""
+        @monitored_clouds.each do |cloud_name|
+          line = "data_source \\\"#{cloud_name}\\\" "
+          ips = []
+          if clouds[cloud_name.intern]
+            clouds[cloud_name.intern].nodes(:status => 'running').each_with_index do |n, i|
+              ips << (n[:private_dns_name] || n[:ip]) + ":8649" # todo - what if we used master0, slave0 etc here?
+            end
+          end
+          data_sources << (line + ips.join(" ") + "\n")
+        end
+        data_sources.gsub!(/\n/, '\n')
+
+        has_variable "ganglia_gmetad_data_sources", :value => data_sources
+
+        # poolname = clouds.values.first.pool.name
+        # has_variable "ganglia_pool_name", :value => (poolname && !poolname.empty? ? poolname : "pool")
+        has_variable "ganglia_pool_name", :value => "pool"
+
+        has_exec(:name => "restart_gmetad2", :command => "/etc/init.d/gmetad restart", :action => :nothing) #  HACK this is already defined!, todo
+        has_file(:name => "/etc/ganglia/gmetad.conf", :mode => 0644, :template => "gmetad.conf.erb" do
+          notifies get_exec("restart_gmetad2"), :run
+        end
+        has_service "gmetad", :enabled => true, :running => true, :supports => [:restart]
+      end
+
+      def track(*features)
+        @monitored_features ||= {}
+        features.map {|f| @monitored_features[f] = true }
+      end
+
+      def gmond_after_all_loaded
+        has_variable "ganglia_cloud_name", :value => cloud_name 
+        has_variable "ganglia_this_nodes_private_ip", :value => lambda{ %Q{%x[curl http://169.254.169.254/latest/meta-data/local-ipv4]}}
+        has_variable "ganglia_masters_ip", :value => lambda { %Q{\`ping -c1  master0 | grep PING | awk -F '[()]' '{print $2 }'\`.strip}}
+
+        first_node = clouds[cloud_name].nodes(:status => 'running').first
+
+
+        if first_node
+          has_variable "ganglia_first_node_in_clusters_ip", :value => lambda { %Q{\`ping -c1  #{first_node[:private_dns_name]} | grep PING | awk -F '[()]' '{print $2 }'\`.strip}}
+
+          has_file(:name => "/etc/ganglia/gmond.conf") do
+            mode 0644
+            template "gmond.conf.erb"
+            # notifies get_exec("restart-gmond"), :run
+          end
+
+          enable_tracking_configs
+
+        end
+        has_service "gmond", :enabled => true, :running => true, :supports => [:restart]
+
+      end
+
+      def enable_tracking_configs
+        if @monitored_features[:hadoop]
+          has_variable "hadoop_ganglia_monitoring_enabled", :value => true
+          
+          # hmm, should maybe be mvd to hadoop plugin?
+          has_file(:name => "/usr/local/hadoop/conf/hadoop-metrics.properties") do
+            mode 0644
+            template "hadoop-metrics.properties.erb"
+          end
+        end
+      end
+
+      def install_extra_gmond_monitors
+        has_directory(:name => "/etc/ganglia/bin/monitors")
+        %w{sshd_ganglia}.each do |monitor|
+          has_file(:name => "/etc/ganglia/bin/monitors/#{monitor}") do
+            mode 0755
+            template "bin/monitors/#{monitor}.sh"
+          end
+          has_cron( :name => "ganglia_monitor_#{monitor}",
+                    :command => "bash -c /etc/ganglia/bin/monitors/#{monitor}",
+                    :user => "root",
+                    :minute => "*/5",
+                    :hour => "*",
+                    :month => "*",
+                    :weekday => "*")
+        end
+      end
+
+      # todo, add a verifier
+      # telnet localhost 8649
+
+    end
+  end
+end
+
+

data/lib/extensions/ganglia/templates/bin/gmetad.erb

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON=/usr/sbin/gmetad
+NAME=gmetad
+DESC="Ganglia Monitor Meta-Daemon"
+
+test -x $DAEMON || exit 0
+set -e
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
+                --exec $DAEMON
+        echo "$NAME."
+        ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        start-stop-daemon --stop  --quiet --oknodo \
+                --exec $DAEMON  2>&1 > /dev/null
+        echo "$NAME."
+        ;;
+  reload)
+  ;;
+  restart|force-reload)
+        $0 stop
+        $0 start
+        ;;
+  *)
+        N=/etc/init.d/$NAME
+        # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+        echo "Usage: $N {start|stop|restart|force-reload}" >&2
+        exit 1
+        ;;
+esac
+
+exit 0

data/lib/extensions/ganglia/templates/bin/gmond.erb

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+DAEMON=/usr/sbin/gmond
+NAME=gmond
+DESC="Ganglia Monitor Daemon"
+
+test -x $DAEMON || exit 0
+set -e
+
+case "$1" in
+  start)
+        echo -n "Starting $DESC: "
+        start-stop-daemon --start --quiet --pidfile /var/run/$NAME.pid \
+                --exec $DAEMON
+        echo "$NAME."
+        ;;
+  stop)
+        echo -n "Stopping $DESC: "
+        start-stop-daemon --stop  --quiet --oknodo \
+                --exec $DAEMON  2>&1 > /dev/null
+        echo "$NAME."
+        ;;
+  reload)
+  ;;
+  restart|force-reload)
+        $0 stop
+        $0 start
+        ;;
+  *)
+        N=/etc/init.d/$NAME
+        # echo "Usage: $N {start|stop|restart|reload|force-reload}" >&2
+        echo "Usage: $N {start|stop|restart|force-reload}" >&2
+        exit 1
+        ;;
+esac
+
+exit 0