edamame 0.2.0

data/lib/edamame/monitoring.rb

@@ -0,0 +1,7 @@
+require 'yaml'
+require 'edamame/monitoring/god_process'
+require 'edamame/monitoring/god_email'
+require 'edamame/monitoring/beanstalkd_god'
+require 'edamame/monitoring/tyrant_god'
+require 'edamame/monitoring/sinatra_god'
+require 'edamame/monitoring/process_groups'

data/lib/edamame/monitoring/README-god.textile

@@ -0,0 +1,54 @@
+Keep running:
+
+* Worker queue (beanstalkd)
+* Backing store (ttserver)
+* Request queue-fed Scrapers (list of scripts)
+* Feed/Periodic scrapers
+* Constant scrapers
+
+* http://god.rubyforge.org/
+* http://railscasts.com/episodes/130-monitoring-with-god
+
+    sudo gem install god
+    god -c config/mailit.god
+    god status
+    god terminate
+    god log mailit-workling
+    kill `cat log/workling.pid`
+
+* http://nubyonrails.com/articles/about-this-blog-beanstalk-messaging-queue
+** The "god.conf":http://pastie.textmate.org/private/ovgxu2ihoicli2ktrwtbew is
+   taken from there.
+
+h2. Beanstalkd
+
+*Usage*:
+
+    beanstalkd  --help
+    Use: beanstalkd [OPTIONS]
+
+    Options:
+     -d       detach
+     -l ADDR  listen on address (default is 0.0.0.0)
+     -p PORT  listen on port (default is 11300)
+     -u USER  become user and group
+     -z SIZE  set the maximum job size in bytes (default is 65535)
+     -v       show version information
+     -h       show this help
+
+
+h2. Tokyo Tyrant
+
+*Usage*:
+
+    ttserver --help
+    ttserver: the server of Tokyo Tyrant
+
+    usage:
+    ttserver [-host name] [-port num] [-thnum num] [-tout num] [-dmn]
+      [-pid path] [-kl] [-log path] [-ld|-le] [-ulog path]
+      [-ulim num] [-uas] [-sid num]
+      [-mhost name] [-mport num] [-rts path] [-rcc] [-skel name]
+      [-ext path] [-extpc name period] [-mask expr] [-unmask expr] [dbname]
+
+

data/lib/edamame/monitoring/beanstalkd_god.rb

@@ -0,0 +1,28 @@
+class BeanstalkdGod < GodProcess
+  BeanstalkdGod::DEFAULT_OPTIONS = {
+    :listen_on      => '0.0.0.0',
+    :port           => 11300,
+    :user           => nil,
+    :max_job_size   => '65535',
+    :max_cpu_usage  => 50.percent,
+    :max_mem_usage  => 500.megabytes,
+    :monitor_group  => 'beanstalkds',
+    :beanstalkd_exe => '/usr/local/bin/beanstalkd',
+  }
+  def self.default_options() super.deep_merge(BeanstalkdGod::DEFAULT_OPTIONS)       ; end
+  def self.site_options()    super.deep_merge(global_site_options[:beanstalkd_god]||{}) ; end
+
+  def self.kind
+    :beanstalkd
+  end
+
+  def start_command
+    [
+      options[:beanstalkd_exe],
+      "-l #{options[:listen_on]}",
+      "-p #{options[:port]}",
+      "-z #{options[:max_job_size]}",
+      options[:user] ? "-u #{options[:user]}" : "",
+    ].flatten.compact.join(" ")
+  end
+end

data/lib/edamame/monitoring/god_email.rb

@@ -0,0 +1,45 @@
+module God
+  def self.setup_email options
+    God::Contacts::Email.message_settings = {
+      :from           => options[:username], }
+    God.contact(:email) do |c|
+      c.name          = options[:to_name]
+      c.email         = options[:to]
+      c.group         = options[:group] || 'default'
+    end
+    if options[:address] then delivery_by_smtp(options)
+    else                     delivery_by_gmail(options) end
+  end
+
+  #
+  # GMail
+  #
+  # http://millarian.com/programming/ruby-on-rails/monitoring-thin-using-god-with-google-apps-notifications/
+  def self.delivery_by_gmail options
+    require 'tlsmail'
+    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+    God::Contacts::Email.server_settings = {
+      :address        => 'smtp.gmail.com',
+      :tls            => 'true',
+      :port           => 587,
+      :domain         => options[:email_domain],
+      :user_name      => options[:username],
+      :password       => options[:password],
+      :authentication => :plain
+    }
+  end
+
+  #
+  # SMTP email
+  #
+  def self.delivery_by_smtp options
+    God::Contacts::Email.server_settings = {
+      :address        => options[:address],
+      :port           => 25,
+      :domain         => options[:email_domain],
+      :user_name      => options[:username],
+      :password       => options[:password],
+      :authentication => :plain,
+    }
+  end
+end

data/lib/edamame/monitoring/god_process.rb

@@ -0,0 +1,205 @@
+class GodProcess
+  DEFAULT_OPTIONS = {
+    :monitor_group      => nil,
+    :uid                => nil,
+    :gid                => nil,
+    :start_notify       => nil,
+    :restart_notify     => nil,
+    :flapping_notify    => nil,
+
+    :process_log_dir    => '/var/log/god',
+
+    :start_grace_time   => 20.seconds,
+    :restart_grace_time => nil,         # start_grace_time+2 if nil
+    :default_interval   => 5.minutes,
+    :start_interval     => 5.minutes,
+    :mem_usage_interval => 20.minutes,
+    :cpu_usage_interval => 20.minutes,
+  }
+  # Site options definition files. Merged in order, later entries win.
+  GLOBAL_SITE_OPTIONS_FILES = []
+  # merged contents of the GLOBAL_SITE_OPTIONS_FILES
+  cattr_reader  :global_site_options
+  attr_accessor :options
+
+  #
+  # * Class options are defined by the edamame code. They define each process'
+  #   base behavior.
+  #
+  # * Site options are defined by config file(s), and define machine/org
+  #   specific policy (paths to daemon executables, for instance). Site options
+  #   override class options.
+  #
+  # * Options passed in at instantiation describe the specifics of this
+  #   particular process -- the path to a database's file, perhaps. They
+  #   override site options (and therefore class options too).
+  #
+  # Note that, though the options hash is preserved, if action
+  #
+  def initialize _options
+    self.options = { }
+    self.options.deep_merge! self.class.default_options
+    self.options.deep_merge! self.class.site_options
+    self.options.deep_merge! _options
+    p self.options
+  end
+
+  #
+  # Walks upwards through the inheritance tree, accumulating default
+  # options. Later (subclass) nodes should override earlier (super) nodes, with
+  # something like
+  #
+  #     def self.default_options
+  #       super.deep_merge(ThisClass::DEFAULT_OPTIONS)
+  #     end
+  #
+  def self.default_options
+    GodProcess::DEFAULT_OPTIONS
+  end
+
+  #
+  # Walks upwards through the inheritance tree, accumulating site
+  # options. Later (subclass) nodes should override earlier (super) nodes, with
+  # something like
+  #
+  #     def self.site_options
+  #       super.deep_merge( global_site_options[:this_class] )
+  #     end
+  #
+  def self.site_options
+    global_site_options[:god_process] || {}
+  end
+
+  def self.global_site_options
+    return @global_site_options if @globalsite_options
+    @global_site_options = {}
+    GLOBAL_SITE_OPTIONS_FILES.each do |options_filename|
+      @global_site_options.deep_merge! YAML.load_file(options_filename)
+    end
+    @global_site_options
+  end
+
+  def setup
+    LOG.info options.inspect
+    God.watch do |watcher|
+      setup_watcher   watcher
+      setup_start     watcher
+      setup_restart   watcher
+      setup_lifecycle watcher
+    end
+  end
+  def self.create options={}
+    proc = self.new options
+    proc.setup
+    proc.mkdirs!
+    proc
+  end
+
+  def handle
+    (options[:handle] || "#{self.class.kind}_#{options[:port]}").to_s
+  end
+
+  # Log file
+  def process_log_file
+    File.join(options[:process_log_dir], handle+".log")
+  end
+
+  # create any directories required by the process
+  def mkdirs!
+    require 'fileutils'
+    FileUtils.mkdir_p File.dirname(process_log_file)
+  end
+
+  # command to start the daemon
+  def start_command
+    options[:start_command]
+  end
+  # command to stop the daemon
+  # return nil to have god daemonize the process
+  def stop_command
+    options[:stop_command]
+  end
+  # command to restart
+  # if stop_command is nil, it lets god daemonize the process
+  # otherwise, by default it runs stop_command, pauses for 1 second, then runs start_command
+  def restart_command
+    return unless stop_command
+    [stop_command, "sleep 1", start_command].join(" && ")
+  end
+
+  #
+  # Setup common to most watchers
+  #
+  def setup_watcher watcher
+    watcher.name             = self.handle
+    watcher.start            = start_command
+    watcher.stop             = stop_command             if stop_command
+    watcher.restart          = restart_command          if restart_command
+    watcher.group            = options[:monitor_group]  if options[:monitor_group]
+    watcher.uid              = options[:uid]            if options[:uid]
+    watcher.gid              = options[:gid]            if options[:gid]
+    watcher.pid_file         = options[:pid_file]       if options[:pid_file]
+    watcher.interval         = options[:default_interval]
+    watcher.start_grace      = options[:start_grace_time]
+    watcher.restart_grace    = options[:restart_grace_time] || (options[:start_grace_time] + 2.seconds)
+    watcher.behavior(:clean_pid_file)
+  end
+
+  #
+  # Starts process
+  #
+  def setup_start watcher
+    watcher.start_if do |start|
+      start.condition(:process_running) do |c|
+        c.interval = options[:start_interval]
+        c.running  = false
+        c.notify     = options[:start_notify] if options[:start_notify]
+      end
+    end
+  end
+
+  #
+  def setup_restart watcher
+    watcher.restart_if do |restart|
+      restart.condition(:memory_usage) do |c|
+        c.interval   = options[:mem_usage_interval] if options[:mem_usage_interval]
+        c.above      = options[:max_mem_usage] || 150.megabytes
+        c.times      = [3, 5] # 3 out of 5 intervals
+        c.notify     = options[:restart_notify] if options[:restart_notify]
+      end
+      restart.condition(:cpu_usage) do |c|
+        c.interval   = options[:cpu_usage_interval] if options[:cpu_usage_interval]
+        c.above      = options[:max_cpu_usage] || 50.percent
+        c.times      = 5
+        c.notify     = options[:restart_notify] if options[:restart_notify]
+      end
+    end
+  end
+
+  # Define lifecycle
+  def setup_lifecycle watcher
+    watcher.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.to_state     = [:start, :restart]
+        c.times        = 10
+        c.within       = 15.minute
+        c.transition   = :unmonitored
+        c.retry_in     = 60.minutes
+        c.retry_times  = 5
+        c.retry_within = 12.hours
+        c.notify       = options[:flapping_notify] if options[:flapping_notify]
+      end
+    end
+  end
+end
+
+class Hash
+  # remove all key-value pairs where the value is nil
+  def compact
+    reject{|key,val| val.nil? }
+  end
+  # Replace the hash with its compacted self
+  def compact!
+    replace(compact)
+  end
+end

data/lib/edamame/monitoring/process_groups.rb

@@ -0,0 +1,32 @@
+module God
+  #
+  # Given a base port number and an associative array
+  #   [ [GodProcessSubclass, { :options => 'for factory methods', ... }],
+  #     ..., }
+  # this creates each given service with incrementing port numbers.
+  #
+  # For example,
+  #
+  #     God.service_group 12300, [
+  #       [BeanstalkdGod, { :max_mem_usage => 2.gigabytes,  }],
+  #       [TyrantGod,     { :db_dirname => EDAMAME_DB_DIR, :db_name => 'queue_jobs.tch' }],
+  #       [TyrantGod,     { :db_dirname => EDAMAME_DB_DIR, :db_name => 'fetched_urls.tch' }],
+  #       [ThinGod,       { :thin_config_yml => '/slice/www/edamame_monitor/current/config.yml' }],
+  #       ]
+  #
+  # will create an edamame pair of beanstalkd queue on 123000 and tyrant DB on
+  # 12301, an app-specific DB on 12302, and a lightweight monitoring web app on
+  # 12303.
+  #
+  # It's up to you to choose the ports to not overlap with other groups, etc.
+  #
+  # If an explicit port is given, that port is used with no regard to the rest
+  # of the group, and its number is skipped.
+  #
+  def self.process_group base_port, services
+    services.each do |klass, options|
+      klass.create({ :port => base_port }.deep_merge(options))
+      base_port += 1
+    end
+  end
+end

data/lib/edamame/monitoring/sinatra_god.rb

@@ -0,0 +1,34 @@
+class SinatraGod < GodProcess
+  SinatraGod::DEFAULT_OPTIONS = {
+    :monitor_group   => 'sinatras',
+    :server_exe      => '/usr/bin/thin',        # path to thin. Override this in the site config file.
+    :port            => 12000,
+    :thin_config_yml => '/somedir/config.yml',
+    :pid_file        => '/var/run/god/sinatra.pid'
+  }
+  def self.default_options() super.deep_merge(SinatraGod::DEFAULT_OPTIONS)           ; end
+  def self.site_options()    super.deep_merge(global_site_options[:sinatra_god]||{}) ; end
+
+  def self.kind
+    :sinatra
+  end
+
+  def thin_command action
+    [ options[:server_exe], action,
+      "--config=#{options[:thin_config_yml]}",
+      (options[:pid_file] ? "--pid=#{options[:pid_file]}" : ''),
+    ].flatten.compact.join(" ")
+  end
+
+  def start_command
+    thin_command :start
+  end
+
+  def restart_command
+    thin_command :restart
+  end
+
+  def stop_command
+    thin_command :stop
+  end
+end

data/lib/edamame/monitoring/tyrant_god.rb

@@ -0,0 +1,59 @@
+#
+# -host   name        : specify the host name or the address of the server.  By default, every network address is bound.
+# -port   num         : specify the port number.  By default, it is 1978.
+#
+# -thnum  num         : specify the number of worker threads.  By default, it is 8.
+# -tout   num         : specify the timeout of each session in seconds.  By default, no timeout is specified.
+#
+# -log    path        : output log messages into the file.
+# -ld                 : log debug messages also.
+# -le                 : log error messages only.
+# -ulog   path        : specify the update log directory.
+# -ulim   num         : specify the limit size of each update log file.
+# -uas                : use asynchronous I/O for the update log.
+#
+# -sid    num         : specify the server ID.
+# -mhost  name        : specify the host name of the replication master server.
+# -mport  num         : specify the port number of the replication master server.
+# -rts    path        : specify the replication time stamp file.
+# -rcc                : check consistency of replication.
+#
+# -skel   name        : specify the name of the skeleton database library.
+# -ext    path        : specify the script language extension file.
+# -extpc  name period : specify the function name and the calling period of a periodic command.
+# -mask   expr        : specify the names of forbidden commands.
+# -unmask expr        : specify the names of allowed commands.
+#
+class TyrantGod < GodProcess
+  TyrantGod::DEFAULT_OPTIONS = {
+    :listen_on      => '0.0.0.0',
+    :port           => 11200,
+    :db_dirname     => '/tmp',
+    #
+    :max_cpu_usage  => 50.percent,
+    :max_mem_usage  => 150.megabytes,
+    :monitor_group  => 'tyrants',
+    :server_exe     => '/usr/local/bin/ttserver',
+  }
+  def self.default_options() super.deep_merge(TyrantGod::DEFAULT_OPTIONS)           ; end
+  def self.site_options()    super.deep_merge(global_site_options[:tyrant_god]||{}) ; end
+
+  def self.kind
+    :ttyrant
+  end
+
+  def dbname
+    basename = options[:db_name] || (handle+'.tct')
+    File.join(options[:db_dirname], basename)
+  end
+
+  def start_command
+    [
+      options[:server_exe],
+      "-host #{options[:listen_on]}",
+      "-port #{options[:port]}",
+      "-log  #{process_log_file}",
+      dbname
+    ].flatten.compact.join(" ")
+  end
+end