edamame 0.3.0 → 0.3.1

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: edamame
 version: !ruby/object:Gem::Version 
+  hash: 17
   prerelease: false
   segments: 
   - 0
   - 3
-  - 0
-  version: 0.3.0
+  - 1
+  version: 0.3.1
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -14,16 +15,18 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-04-13 00:00:00 +00:00
+date: 2010-08-16 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: beanstalk-client
   prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -33,9 +36,11 @@ dependencies:
   name: wukong
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -45,9 +50,11 @@ dependencies:
   name: monkeyshines
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 3
         segments: 
         - 0
         version: "0"
@@ -94,13 +101,6 @@ files:
 - lib/edamame/broker.rb
 - lib/edamame/job.rb
 - lib/edamame/monitoring.rb
-- lib/edamame/monitoring/README-god.textile
-- lib/edamame/monitoring/beanstalkd_god.rb
-- lib/edamame/monitoring/god_email.rb
-- lib/edamame/monitoring/god_process.rb
-- lib/edamame/monitoring/process_groups.rb
-- lib/edamame/monitoring/sinatra_god.rb
-- lib/edamame/monitoring/tyrant_god.rb
 - lib/edamame/persistent_queue.rb
 - lib/edamame/queue.rb
 - lib/edamame/queue/beanstalk.rb
@@ -134,26 +134,30 @@ rdoc_options:
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: 3
       segments: 
       - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.3.6
+rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
 summary: Beanstalk + Tokyo Tyrant = Edamame, a fast persistent distributed priority job queue.
 test_files: 
-- spec/edamame_spec.rb
 - spec/spec_helper.rb
+- spec/edamame_spec.rb

data/lib/edamame/monitoring/README-god.textile

@@ -1,54 +0,0 @@
-Keep running:
-
-* Worker queue (beanstalkd)
-* Backing store (ttserver)
-* Request queue-fed Scrapers (list of scripts)
-* Feed/Periodic scrapers
-* Constant scrapers
-
-* http://god.rubyforge.org/
-* http://railscasts.com/episodes/130-monitoring-with-god
-
-    sudo gem install god
-    god -c config/mailit.god
-    god status
-    god terminate
-    god log mailit-workling
-    kill `cat log/workling.pid`
-
-* http://nubyonrails.com/articles/about-this-blog-beanstalk-messaging-queue
-** The "god.conf":http://pastie.textmate.org/private/ovgxu2ihoicli2ktrwtbew is
-   taken from there.
-
-h2. Beanstalkd
-
-*Usage*:
-
-    beanstalkd  --help
-    Use: beanstalkd [OPTIONS]
-
-    Options:
-     -d       detach
-     -l ADDR  listen on address (default is 0.0.0.0)
-     -p PORT  listen on port (default is 11300)
-     -u USER  become user and group
-     -z SIZE  set the maximum job size in bytes (default is 65535)
-     -v       show version information
-     -h       show this help
-
-
-h2. Tokyo Tyrant
-
-*Usage*:
-
-    ttserver --help
-    ttserver: the server of Tokyo Tyrant
-
-    usage:
-    ttserver [-host name] [-port num] [-thnum num] [-tout num] [-dmn]
-      [-pid path] [-kl] [-log path] [-ld|-le] [-ulog path]
-      [-ulim num] [-uas] [-sid num]
-      [-mhost name] [-mport num] [-rts path] [-rcc] [-skel name]
-      [-ext path] [-extpc name period] [-mask expr] [-unmask expr] [dbname]
-
-

data/lib/edamame/monitoring/beanstalkd_god.rb

@@ -1,28 +0,0 @@
-class BeanstalkdGod < GodProcess
-  BeanstalkdGod::DEFAULT_OPTIONS = {
-    :listen_on      => '0.0.0.0',
-    :port           => 11300,
-    :user           => nil,
-    :max_job_size   => '65535',
-    :max_cpu_usage  => 50.percent,
-    :max_mem_usage  => 500.megabytes,
-    :monitor_group  => 'beanstalkds',
-    :beanstalkd_exe => '/usr/local/bin/beanstalkd',
-  }
-  def self.default_options() super.deep_merge(BeanstalkdGod::DEFAULT_OPTIONS)       ; end
-  def self.site_options()    super.deep_merge(global_site_options[:beanstalkd_god]||{}) ; end
-
-  def self.kind
-    :beanstalkd
-  end
-
-  def start_command
-    [
-      options[:beanstalkd_exe],
-      "-l #{options[:listen_on]}",
-      "-p #{options[:port]}",
-      "-z #{options[:max_job_size]}",
-      options[:user] ? "-u #{options[:user]}" : "",
-    ].flatten.compact.join(" ")
-  end
-end

data/lib/edamame/monitoring/god_email.rb

@@ -1,45 +0,0 @@
-module God
-  def self.setup_email options
-    God::Contacts::Email.message_settings = {
-      :from           => options[:username], }
-    God.contact(:email) do |c|
-      c.name          = options[:to_name]
-      c.email         = options[:to]
-      c.group         = options[:group] || 'default'
-    end
-    if options[:address] then delivery_by_smtp(options)
-    else                     delivery_by_gmail(options) end
-  end
-
-  #
-  # GMail
-  #
-  # http://millarian.com/programming/ruby-on-rails/monitoring-thin-using-god-with-google-apps-notifications/
-  def self.delivery_by_gmail options
-    require 'tlsmail'
-    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
-    God::Contacts::Email.server_settings = {
-      :address        => 'smtp.gmail.com',
-      :tls            => 'true',
-      :port           => 587,
-      :domain         => options[:email_domain],
-      :user_name      => options[:username],
-      :password       => options[:password],
-      :authentication => :plain
-    }
-  end
-
-  #
-  # SMTP email
-  #
-  def self.delivery_by_smtp options
-    God::Contacts::Email.server_settings = {
-      :address        => options[:address],
-      :port           => 25,
-      :domain         => options[:email_domain],
-      :user_name      => options[:username],
-      :password       => options[:password],
-      :authentication => :plain,
-    }
-  end
-end

data/lib/edamame/monitoring/god_process.rb

@@ -1,205 +0,0 @@
-class GodProcess
-  DEFAULT_OPTIONS = {
-    :monitor_group      => nil,
-    :uid                => nil,
-    :gid                => nil,
-    :start_notify       => nil,
-    :restart_notify     => nil,
-    :flapping_notify    => nil,
-
-    :process_log_dir    => '/var/log/god',
-
-    :start_grace_time   => 20.seconds,
-    :restart_grace_time => nil,         # start_grace_time+2 if nil
-    :default_interval   => 5.minutes,
-    :start_interval     => 5.minutes,
-    :mem_usage_interval => 20.minutes,
-    :cpu_usage_interval => 20.minutes,
-  }
-  # Site options definition files. Merged in order, later entries win.
-  GLOBAL_SITE_OPTIONS_FILES = []
-  # merged contents of the GLOBAL_SITE_OPTIONS_FILES
-  cattr_reader  :global_site_options
-  attr_accessor :options
-
-  #
-  # * Class options are defined by the edamame code. They define each process'
-  #   base behavior.
-  #
-  # * Site options are defined by config file(s), and define machine/org
-  #   specific policy (paths to daemon executables, for instance). Site options
-  #   override class options.
-  #
-  # * Options passed in at instantiation describe the specifics of this
-  #   particular process -- the path to a database's file, perhaps. They
-  #   override site options (and therefore class options too).
-  #
-  # Note that, though the options hash is preserved, if action
-  #
-  def initialize _options
-    self.options = { }
-    self.options.deep_merge! self.class.default_options
-    self.options.deep_merge! self.class.site_options
-    self.options.deep_merge! _options
-    p self.options
-  end
-
-  #
-  # Walks upwards through the inheritance tree, accumulating default
-  # options. Later (subclass) nodes should override earlier (super) nodes, with
-  # something like
-  #
-  #     def self.default_options
-  #       super.deep_merge(ThisClass::DEFAULT_OPTIONS)
-  #     end
-  #
-  def self.default_options
-    GodProcess::DEFAULT_OPTIONS
-  end
-
-  #
-  # Walks upwards through the inheritance tree, accumulating site
-  # options. Later (subclass) nodes should override earlier (super) nodes, with
-  # something like
-  #
-  #     def self.site_options
-  #       super.deep_merge( global_site_options[:this_class] )
-  #     end
-  #
-  def self.site_options
-    global_site_options[:god_process] || {}
-  end
-
-  def self.global_site_options
-    return @global_site_options if @globalsite_options
-    @global_site_options = {}
-    GLOBAL_SITE_OPTIONS_FILES.each do |options_filename|
-      @global_site_options.deep_merge! YAML.load_file(options_filename)
-    end
-    @global_site_options
-  end
-
-  def setup
-    LOG.info options.inspect
-    God.watch do |watcher|
-      setup_watcher   watcher
-      setup_start     watcher
-      setup_restart   watcher
-      setup_lifecycle watcher
-    end
-  end
-  def self.create options={}
-    proc = self.new options
-    proc.setup
-    proc.mkdirs!
-    proc
-  end
-
-  def handle
-    (options[:handle] || "#{self.class.kind}_#{options[:port]}").to_s
-  end
-
-  # Log file
-  def process_log_file
-    File.join(options[:process_log_dir], handle+".log")
-  end
-
-  # create any directories required by the process
-  def mkdirs!
-    require 'fileutils'
-    FileUtils.mkdir_p File.dirname(process_log_file)
-  end
-
-  # command to start the daemon
-  def start_command
-    options[:start_command]
-  end
-  # command to stop the daemon
-  # return nil to have god daemonize the process
-  def stop_command
-    options[:stop_command]
-  end
-  # command to restart
-  # if stop_command is nil, it lets god daemonize the process
-  # otherwise, by default it runs stop_command, pauses for 1 second, then runs start_command
-  def restart_command
-    return unless stop_command
-    [stop_command, "sleep 1", start_command].join(" && ")
-  end
-
-  #
-  # Setup common to most watchers
-  #
-  def setup_watcher watcher
-    watcher.name             = self.handle
-    watcher.start            = start_command
-    watcher.stop             = stop_command             if stop_command
-    watcher.restart          = restart_command          if restart_command
-    watcher.group            = options[:monitor_group]  if options[:monitor_group]
-    watcher.uid              = options[:uid]            if options[:uid]
-    watcher.gid              = options[:gid]            if options[:gid]
-    watcher.pid_file         = options[:pid_file]       if options[:pid_file]
-    watcher.interval         = options[:default_interval]
-    watcher.start_grace      = options[:start_grace_time]
-    watcher.restart_grace    = options[:restart_grace_time] || (options[:start_grace_time] + 2.seconds)
-    watcher.behavior(:clean_pid_file)
-  end
-
-  #
-  # Starts process
-  #
-  def setup_start watcher
-    watcher.start_if do |start|
-      start.condition(:process_running) do |c|
-        c.interval = options[:start_interval]
-        c.running  = false
-        c.notify     = options[:start_notify] if options[:start_notify]
-      end
-    end
-  end
-
-  #
-  def setup_restart watcher
-    watcher.restart_if do |restart|
-      restart.condition(:memory_usage) do |c|
-        c.interval   = options[:mem_usage_interval] if options[:mem_usage_interval]
-        c.above      = options[:max_mem_usage] || 150.megabytes
-        c.times      = [3, 5] # 3 out of 5 intervals
-        c.notify     = options[:restart_notify] if options[:restart_notify]
-      end
-      restart.condition(:cpu_usage) do |c|
-        c.interval   = options[:cpu_usage_interval] if options[:cpu_usage_interval]
-        c.above      = options[:max_cpu_usage] || 50.percent
-        c.times      = 5
-        c.notify     = options[:restart_notify] if options[:restart_notify]
-      end
-    end
-  end
-
-  # Define lifecycle
-  def setup_lifecycle watcher
-    watcher.lifecycle do |on|
-      on.condition(:flapping) do |c|
-        c.to_state     = [:start, :restart]
-        c.times        = 10
-        c.within       = 15.minute
-        c.transition   = :unmonitored
-        c.retry_in     = 60.minutes
-        c.retry_times  = 5
-        c.retry_within = 12.hours
-        c.notify       = options[:flapping_notify] if options[:flapping_notify]
-      end
-    end
-  end
-end
-
-class Hash
-  # remove all key-value pairs where the value is nil
-  def compact
-    reject{|key,val| val.nil? }
-  end
-  # Replace the hash with its compacted self
-  def compact!
-    replace(compact)
-  end
-end

data/lib/edamame/monitoring/process_groups.rb

@@ -1,32 +0,0 @@
-module God
-  #
-  # Given a base port number and an associative array
-  #   [ [GodProcessSubclass, { :options => 'for factory methods', ... }],
-  #     ..., }
-  # this creates each given service with incrementing port numbers.
-  #
-  # For example,
-  #
-  #     God.service_group 12300, [
-  #       [BeanstalkdGod, { :max_mem_usage => 2.gigabytes,  }],
-  #       [TyrantGod,     { :db_dirname => EDAMAME_DB_DIR, :db_name => 'queue_jobs.tch' }],
-  #       [TyrantGod,     { :db_dirname => EDAMAME_DB_DIR, :db_name => 'fetched_urls.tch' }],
-  #       [ThinGod,       { :thin_config_yml => '/slice/www/edamame_monitor/current/config.yml' }],
-  #       ]
-  #
-  # will create an edamame pair of beanstalkd queue on 123000 and tyrant DB on
-  # 12301, an app-specific DB on 12302, and a lightweight monitoring web app on
-  # 12303.
-  #
-  # It's up to you to choose the ports to not overlap with other groups, etc.
-  #
-  # If an explicit port is given, that port is used with no regard to the rest
-  # of the group, and its number is skipped.
-  #
-  def self.process_group base_port, services
-    services.each do |klass, options|
-      klass.create({ :port => base_port }.deep_merge(options))
-      base_port += 1
-    end
-  end
-end

data/lib/edamame/monitoring/sinatra_god.rb

@@ -1,34 +0,0 @@
-class SinatraGod < GodProcess
-  SinatraGod::DEFAULT_OPTIONS = {
-    :monitor_group   => 'sinatras',
-    :server_exe      => '/usr/bin/thin',        # path to thin. Override this in the site config file.
-    :port            => 12000,
-    :thin_config_yml => '/somedir/config.yml',
-    :pid_file        => '/var/run/god/sinatra.pid'
-  }
-  def self.default_options() super.deep_merge(SinatraGod::DEFAULT_OPTIONS)           ; end
-  def self.site_options()    super.deep_merge(global_site_options[:sinatra_god]||{}) ; end
-
-  def self.kind
-    :sinatra
-  end
-
-  def thin_command action
-    [ options[:server_exe], action,
-      "--config=#{options[:thin_config_yml]}",
-      (options[:pid_file] ? "--pid=#{options[:pid_file]}" : ''),
-    ].flatten.compact.join(" ")
-  end
-
-  def start_command
-    thin_command :start
-  end
-
-  def restart_command
-    thin_command :restart
-  end
-
-  def stop_command
-    thin_command :stop
-  end
-end

data/lib/edamame/monitoring/tyrant_god.rb

@@ -1,65 +0,0 @@
-#
-# -host   name        : specify the host name or the address of the server.  By default, every network address is bound.
-# -port   num         : specify the port number.  By default, it is 1978.
-#
-# -thnum  num         : specify the number of worker threads.  By default, it is 8.
-# -tout   num         : specify the timeout of each session in seconds.  By default, no timeout is specified.
-#
-# -log    path        : output log messages into the file.
-# -ld                 : log debug messages also.
-# -le                 : log error messages only.
-# -ulog   path        : specify the update log directory.
-# -ulim   num         : specify the limit size of each update log file.
-# -uas                : use asynchronous I/O for the update log.
-#
-# -sid    num         : specify the server ID.
-# -mhost  name        : specify the host name of the replication master server.
-# -mport  num         : specify the port number of the replication master server.
-# -rts    path        : specify the replication time stamp file.
-# -rcc                : check consistency of replication.
-#
-# -skel   name        : specify the name of the skeleton database library.
-# -ext    path        : specify the script language extension file.
-# -extpc  name period : specify the function name and the calling period of a periodic command.
-# -mask   expr        : specify the names of forbidden commands.
-# -unmask expr        : specify the names of allowed commands.
-#
-class TyrantGod < GodProcess
-  TyrantGod::DEFAULT_OPTIONS = {
-    :listen_on      => '0.0.0.0',
-    :port           => 11200,
-    :db_dirname     => '/tmp',
-    #
-    :max_cpu_usage  => 50.percent,
-    :max_mem_usage  => 150.megabytes,
-    :monitor_group  => 'tyrants',
-    :server_exe     => '/usr/local/bin/ttserver',
-  }
-  def self.default_options() super.deep_merge(TyrantGod::DEFAULT_OPTIONS)           ; end
-  def self.site_options()    super.deep_merge(global_site_options[:tyrant_god]||{}) ; end
-
-  def self.kind
-    :ttyrant
-  end
-
-  def dbname
-    basename = options[:db_name] || (handle+'.tct')
-    File.join(options[:db_dirname], basename)
-  end
-
-  # create any directories required by the process
-  def mkdirs!
-    super
-    FileUtils.mkdir_p File.dirname(dbname)
-  end
-
-  def start_command
-    [
-      options[:server_exe],
-      "-host #{options[:listen_on]}",
-      "-port #{options[:port]}",
-      "-log  #{process_log_file}",
-      dbname
-    ].flatten.compact.join(" ")
-  end
-end