godhead 0.0.2 → 0.0.3

data/README.textile

@@ -0,0 +1,50 @@
+h1. Godhead: Recipes for Monitoring with God
+
+Godhead lets you configure God monitoring tasks to separate configuration of _the program being monitored_ from configuration of _the specific project it's used in_.
+
+It also contains a collection of starting recipes gathered from around the web:
+
+* Nginx
+* Mongrel
+* Thin
+* Unicorn
+* Beanstalkd
+* Starling
+* Memcached
+* Tokyo Tyrant
+* Generic worker daemons
+
+There's skeleton files for delayed_job, workling
+
+h2. Dependencies
+
+Requires the gems for active_support and, of course, god. Optionally requires tlsmail (for Gmail or other secure email notification).
+
+If you enjoy godhead you may also wish to find  "Jesus,":http://github.com/dmathieu/jesus the "web interface for god to speak with mankind". Allelujah.
+
+h2. Examples
+
+The @examples/@ directory holds:
+* a script to run god as a service.
+** @sudo cp [GODHEAD]/examples/etc/init.d/god /etc/init.d/god@
+** @sudo chmod a+x /etc/init.d/god@
+** @update-rc.d god defaults@
+* a god.conf that loads God, sets up notifications, and then runs all .god files in /etc/god
+* an example project script to put into /etc/god
+
+h2. Our use case for godhead
+
+At infochimps, we have a lot of boxen set up as "scrapers":http://github.com/mrflip/monkeyshines and as "feed endpoints.":http://github.com/mrflip/flood_monkey These typically talk to a lightweight database of some sort, pull or push to a queue, and are controlled through a simple sinatra admin panel.
+
+With Godhead we describe the platonic form of, say, a "god config to monitor Tokyo Tyrant".  The god files for a project using Tokyo Tyrant specifies only those few ways (port numbers, memory limits, etc) that its tasks differ from any other project's Tokyo Tyrant tasks. Configuration is still done in Ruby, through any combination of config params and subclassing.
+
+h2. Links
+
+* "God Monitoring homepage":http://god.rubyforge.org/
+* "Railscast with God monitoring Workling and Starling":http://railscasts.com/episodes/130-monitoring-with-god
+* "beanstalkd"
+** "article by topfunky":http://nubyonrails.com/articles/about-this-blog-beanstalk-messaging-queue
+** "god recipe":http://pastie.textmate.org/private/ovgxu2ihoicli2ktrwtbew 
+* "delayed_job":http://github.com/blog/229-dj-god
+* "init.d/god script":http://www.johnwulff.com/2008/04/30/deploy-typo-using-capistrano
+* backgroundRb: "this one":http://effectif.com/articles/monitoring-backgroundrb-with-god - or "this one":http://blog.jonathanrwallace.com/2008/08/monitoring-backgroundrb-workers-with-god/

data/Rakefile

@@ -11,8 +11,8 @@ begin
     gem.homepage = "http://github.com/mrflip/godhead"
     gem.authors = ["Philip (flip) Kromer"]
     gem.add_development_dependency "rspec"
-    gem.add_development_dependency "yard"
-    gem.add_dependency 'extlib'
+    gem.add_dependency 'god'
+    gem.add_dependency 'activesupport'
     # gem is a Gem::Specification... see http://www.rubygems.org/read/chapter/20 for additional settings
   end
 rescue LoadError
@@ -32,14 +32,4 @@ Spec::Rake::SpecTask.new(:rcov) do |spec|
 end
 
 task :spec => :check_dependencies
-
 task :default => :spec
-
-begin
-  require 'yard'
-  YARD::Rake::YardocTask.new
-rescue LoadError
-  task :yardoc do
-    abort "YARD is not available. In order to run yardoc, you must: sudo gem install yard"
-  end
-end

data/VERSION

@@ -1 +1 @@
-0.0.2
+0.0.3

data/examples/etc/god.conf

@@ -0,0 +1,14 @@
+require 'rubygems'
+# require 'active_support'
+require 'extlib'
+require 'godhead'
+GOD_CONFIG = YAML.load(File.open('/etc/god/god_config.yaml')) rescue {}
+
+# set up email
+require 'godhead/notifications'
+God.notify_by_gmail GOD_CONFIG[:email]
+God.add_contact 'default',   'coders@example.com'
+God.add_contact 'only_biff', 'biff@example.com'  
+
+# load in all god configs
+God.load "/etc/god/*.god"

data/examples/etc/god/god_config.yaml

@@ -0,0 +1,10 @@
+--- # -*- YAML -*-
+#
+# Save this file in your god dir (probably /etc/god), *then* change the settings
+# below.  Make sure your version control system is set to ignore the file.
+#
+
+:email:
+  :username:            robot@mydomain.com
+  :password:            password1your_mom
+  :smtp_domain:         mydomain.com

data/examples/etc/god/sample_project.god

@@ -0,0 +1,31 @@
+YUPFRONT_ROOT = "/var/www/yuploader"
+
+# ===========================================================================
+#
+# A worker daemon watching a starling queue
+#
+group_options = { :monitor_group => :yupshot, :stop_notify => :only_flip, :flapping_notify => :only_flip }
+
+Godhead::MemcachedRecipe.create     group_options.merge({
+    :user => 'www-data', :pid_file => '/tmp/mc.pid'})
+Godhead::StarlingRecipe.create      group_options
+Godhead::GenericWorkerRecipe.create group_options.merge({
+    :runner_path => "#{YUPFRONT_ROOT}/yupshot/bin/yupshot_worker_daemon" })
+
+# ===========================================================================
+#
+# The frontend server that accepts jobs and inserts them into the queue
+#
+group_options = { :monitor_group => :yupfront , :flapping_notify => :default }}
+
+# nginx front end
+# if nginx feeds backend servers for many different projects, move it to its own file
+Godhead::NginxRunnerRecipe.create group_options.merge({ })
+
+# thin runs clients
+(5000..5003).each do |port|
+  Godhead::ThinRecipe.create(group_options.merge({
+      :port        => port,
+      :rackup_file => File.join(YUPFRONT_ROOT, 'yupfront/config.ru'),
+      :runner_conf => File.join(YUPFRONT_ROOT, 'yupfront/production.yml') }))
+end

data/examples/etc/init.d/god

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+#
+# God init.d (startup) script 
+#
+# save this file as /etc/init.d/god
+# make it executable:
+#    chmod +x /etc/init.d/god
+# Tell the os to run the script at startup:
+#    sudo /usr/sbin/update-rc.d -f god defaults
+#
+# Also consider adding this line (kills god weekly) to your crontab (sudo crontab -e):
+#
+#    # deicide is painless
+#    0 1 * * 0 god quit; sleep 1; killall god; sleep 1; killall -9 god; sleep 1; /etc/init.d/god start
+# 
+
+RETVAL=0
+god_conf="/etc/god.conf"
+god_pid_file="/var/run/god/god.pid" ; mkdir -p `dirname $god_pid_file`
+god_log_file="/var/log/god/god.log" ; mkdir -p `dirname $god_log_file`
+case "$1" in
+  start)
+    god -c "$god_conf" -P "$god_pid_file" -l "$god_log_file"
+    RETVAL=$?
+    echo "God started"
+    ;;
+  stop)
+    kill `cat $god_pid_file`
+    RETVAL=$?
+    echo "God stopped"
+    ;;
+  restart)
+    kill `cat $god_pid_file`
+    god -c "$god_conf" -P "$god_pid_file" -l "$god_log_file"
+    RETVAL=$?
+    echo "God restarted"
+    ;;
+  status)
+    RETVAL=$?
+    ;;
+  *)
+    echo "Usage: god {start|stop|restart|status}"
+    exit 1
+    ;;
+esac
+
+exit $RETVAL

data/godhead.gemspec

@@ -1,47 +1,58 @@
 # Generated by jeweler
-# DO NOT EDIT THIS FILE
-# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{godhead}
-  s.version = "0.0.2"
+  s.version = "0.0.3"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2009-12-07}
+  s.date = %q{2010-02-11}
   s.description = %q{Configure God monitored processes according to their concerns the servers (path and so forth), site policy (number, ports, etc), and notification (email groups, mailserver, etc).}
   s.email = %q{flip@infochimps.org}
   s.extra_rdoc_files = [
     "LICENSE",
-     "README-god.textile"
+     "README.textile"
   ]
   s.files = [
     ".document",
      ".gitignore",
      "LICENSE",
-     "README-god.textile",
+     "README.textile",
      "Rakefile",
      "VERSION",
+     "examples/etc/god.conf",
+     "examples/etc/god/god_config.yaml",
+     "examples/etc/god/sample_project.god",
+     "examples/etc/init.d/god",
      "godhead.gemspec",
      "lib/godhead.rb",
      "lib/godhead/extensions.rb",
      "lib/godhead/extensions/hash.rb",
+     "lib/godhead/extensions/string.rb",
+     "lib/godhead/god_email.rb",
      "lib/godhead/god_recipe.rb",
      "lib/godhead/mixins.rb",
      "lib/godhead/mixins/runs_as_service.rb",
-     "lib/godhead/notification.rb",
-     "lib/godhead/notification/gmail.rb",
+     "lib/godhead/notifications.rb",
      "lib/godhead/process_groups.rb",
      "lib/godhead/recipes.rb",
      "lib/godhead/recipes/beanstalkd_recipe.rb",
+     "lib/godhead/recipes/delayed_job_recipe.rb",
      "lib/godhead/recipes/generic_worker_recipe.rb",
      "lib/godhead/recipes/memcached_recipe.rb",
+     "lib/godhead/recipes/mongrel_recipe.rb",
+     "lib/godhead/recipes/mysql_recipe.rb",
      "lib/godhead/recipes/nginx_recipe.rb",
+     "lib/godhead/recipes/resque_recipe.rb",
      "lib/godhead/recipes/starling_recipe.rb",
      "lib/godhead/recipes/thin_recipe.rb",
+     "lib/godhead/recipes/thinking_sphinx.rb",
      "lib/godhead/recipes/tyrant_recipe.rb",
-     "sample.god",
+     "lib/godhead/recipes/unicorn_recipe.rb",
+     "lib/godhead/recipes/workling_recipe.rb",
      "spec/godhead_spec.rb",
      "spec/spec_helper.rb"
   ]
@@ -61,16 +72,17 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
       s.add_development_dependency(%q<rspec>, [">= 0"])
-      s.add_development_dependency(%q<yard>, [">= 0"])
-      s.add_runtime_dependency(%q<extlib>, [">= 0"])
+      s.add_runtime_dependency(%q<god>, [">= 0"])
+      s.add_runtime_dependency(%q<activesupport>, [">= 0"])
     else
       s.add_dependency(%q<rspec>, [">= 0"])
-      s.add_dependency(%q<yard>, [">= 0"])
-      s.add_dependency(%q<extlib>, [">= 0"])
+      s.add_dependency(%q<god>, [">= 0"])
+      s.add_dependency(%q<activesupport>, [">= 0"])
     end
   else
     s.add_dependency(%q<rspec>, [">= 0"])
-    s.add_dependency(%q<yard>, [">= 0"])
-    s.add_dependency(%q<extlib>, [">= 0"])
+    s.add_dependency(%q<god>, [">= 0"])
+    s.add_dependency(%q<activesupport>, [">= 0"])
   end
 end
+

data/lib/godhead/extensions.rb

@@ -1 +1,2 @@
 require 'godhead/extensions/hash'
+require 'godhead/extensions/string'

data/lib/godhead/extensions/string.rb

@@ -0,0 +1,9 @@
+class String
+  [:camelize, :underscore, :demodulize].each do |inflection|
+    next if self.respond_to?(inflection)
+    self.class_eval %Q{
+      def #{inflection}
+        Extlib::Inflection.#{inflection} self
+      end}
+  end
+end

data/lib/godhead/god_email.rb

@@ -0,0 +1,45 @@
+module God
+  #
+  # Sets up
+  #
+  def self.add_simple_contact to_address, group=nil, &block
+    God.contact(:email) do |c|
+      c.email         = to_address
+      c.group         = group || 'default'
+      block.call(c) if block
+    end
+  end
+
+  #
+  # GMail
+  #
+  # Sends mail using a gmail or google apps account.
+  #
+  # http://millarian.com/programming/ruby-on-rails/monitoring-thin-using-god-with-google-apps-notifications/
+  def self.notify_by_gmail options
+    require 'tlsmail'
+    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+    notify_by_smtp options.reverse_merge({
+        :address        => 'smtp.gmail.com',
+        :tls            => 'true',
+        :port           => 587,
+      })
+  end
+
+  #
+  # SMTP email
+  #
+  # Assumes that the 'from' account and the smtp server login name are identical.
+  def self.notify_by_smtp options
+    God::Contacts::Email.message_settings = {
+      :from           => options[:username], }
+    God::Contacts::Email.server_settings = {
+      :address        => options[:address],
+      :port           => options[:port] || 25,
+      :user_name      => options[:username],
+      :password       => options[:password],
+      :domain         => options[:domain],
+      :authentication => :plain
+    }
+  end
+end

data/lib/godhead/god_recipe.rb

@@ -4,16 +4,18 @@ module Godhead
       :monitor_group      => nil,
       :uid                => nil,
       :gid                => nil,
-      :start_notify       => nil,
+      :crash_notify       => nil,
       :restart_notify     => nil,
       :flapping_notify    => nil,
       :process_log_dir    => '/var/log/god',
-      :pid_dir            => "/var/run/god",
+      :pid_dir            => '/var/run/god',
       :start_grace_time   => 10.seconds,
       :restart_grace_time => nil,         # will be start_grace_time+2 if left nil
       :default_interval   => 5.minutes,
-      :start_interval     => 5.minutes,
+      :start_interval     => nil,         # uses default_interval if unset
+      :max_mem_usage      => nil,         # doesn't monitor mem usage if nil
       :mem_usage_interval => 10.minutes,
+      :max_cpu_usage      => 50.percent,  # doesn't monitor cpu usage if nil
       :cpu_usage_interval => 10.minutes,
     }
     #
@@ -79,9 +81,13 @@ module Godhead
     def setup_start watcher
       watcher.start_if do |start|
         start.condition(:process_running) do |c|
-          c.interval = options[:start_interval]
+          c.interval = options[:start_interval] || options[:default_interval]
           c.running  = false
-          c.notify     = options[:start_notify] if options[:start_notify]
+        end
+      end
+      watcher.transition(:up, :start) do |on|
+        on.condition(:process_exits) do |c|
+          c.notify   = options[:crash_notify] if options[:crash_notify]
         end
       end
     end
@@ -91,16 +97,16 @@ module Godhead
       watcher.restart_if do |restart|
         restart.condition(:memory_usage) do |c|
           c.interval   = options[:mem_usage_interval] if options[:mem_usage_interval]
-          c.above      = options[:max_mem_usage] || 150.megabytes
+          c.above      = options[:max_mem_usage]
           c.times      = [3, 5] # 3 out of 5 intervals
           c.notify     = options[:restart_notify] if options[:restart_notify]
-        end
+        end if options[:max_mem_usage]
         restart.condition(:cpu_usage) do |c|
           c.interval   = options[:cpu_usage_interval] if options[:cpu_usage_interval]
-          c.above      = options[:max_cpu_usage] || 50.percent
+          c.above      = options[:max_cpu_usage]
           c.times      = 5
           c.notify     = options[:restart_notify] if options[:restart_notify]
-        end
+        end if options[:max_cpu_usage]
       end
     end
 
@@ -109,12 +115,12 @@ module Godhead
       watcher.lifecycle do |on|
         on.condition(:flapping) do |c|
           c.to_state     = [:start, :restart]
-          c.times        = 10
-          c.within       = 15.minute
+          c.times        = 5
+          c.within       = options[:flapping_window]   || 15.minute
           c.transition   = :unmonitored
-          c.retry_in     = 60.minutes
+          c.retry_in     = options[:flapping_retry_in] || 30.minutes
           c.retry_times  = 5
-          c.retry_within = 12.hours
+          c.retry_within = 4.hours
           c.notify       = options[:flapping_notify] if options[:flapping_notify]
         end
       end
@@ -187,6 +193,7 @@ module Godhead
 
     # by default, uses :pid_dir/:recipe_name_:port.pid
     def pid_file
+      return nil if (options[:pid_file] == false)
       options[:pid_file] || File.join(options[:pid_dir], "#{recipe_name}_#{options[:port]}.pid")
     end
 

data/lib/godhead/notifications.rb

@@ -0,0 +1,43 @@
+module God
+  #
+  # Adds a notification contact
+  # pass in a group name and a valid email address.
+  #
+  def self.add_contact group, to_address
+    God.contact(:email) do |c|
+      c.group         = group
+      c.email         = to_address
+      c.name          = "God monitoring #{group} notification"
+    end
+  end
+
+  #
+  # GMail
+  #
+  # http://millarian.com/programming/ruby-on-rails/monitoring-thin-using-god-with-google-apps-notifications/
+  def self.notify_by_gmail options
+    require 'tlsmail'
+    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+    notify_by_smtp options.reverse_merge({
+        :address        => 'smtp.gmail.com',
+        :tls            => 'true',
+        :port           => 587,
+      })
+  end
+
+  #
+  # SMTP email
+  #
+  def self.notify_by_smtp options
+    God::Contacts::Email.message_settings = {
+      :from           => options[:username], }
+    God::Contacts::Email.server_settings = {
+      :address        => options[:address],
+      :port           => options[:port] || 25,
+      :domain         => options[:smtp_domain],
+      :user_name      => options[:username],
+      :password       => options[:password],
+      :authentication => :plain,
+    }
+  end
+end