mrflip-edamame 0.1.1

data/lib/edamame/store/tyrant_store.rb

@@ -0,0 +1,50 @@
+require 'tokyotyrant'
+module Edamame
+  module Store
+
+    #
+    # Implementation of KeyStore with a Local TokyoCabinet table database (TDB)
+    #
+    class TyrantStore < Edamame::Store::Base
+      attr_accessor :db_host, :db_port
+
+      # pass in the host:port uri of the key store.
+      def initialize options
+        self.db_host, self.db_port = options[:uri].to_s.split(':')
+        super options
+      end
+
+      def db
+        return @db if @db
+        @db ||= TokyoTyrant::RDBTBL.new
+        @db.open(db_host, db_port) or raise("Can't open DB #{db_host}:#{db_port}. Pass in host:port' #{@db.ecode}: #{@db.errmsg(@db.ecode)}")
+        @db
+      end
+
+      def close
+        @db.close if @db
+        @db = nil
+      end
+
+      # Save the value into the database without waiting for a response.
+      def set_nr(key, val)
+        db.putnr key, val if val
+      end
+
+      def size()
+        db.rnum
+      end
+
+      def stats
+        {  :size => size }
+      end
+
+      def include? *args
+        db.has_key? *args
+      end
+
+    end #class
+
+  end
+end
+

data/lib/methods.txt

@@ -0,0 +1,94 @@
+See also:
+
+  http://github.com/kr/beanstalkd/blob/master/doc/protocol.txt
+
+== Connection == 
+
+connection.rb:  	def initialize(addr, default_tube=nil)
+connection.rb:  	def connect
+connection.rb:  	def close
+
+connection.rb:  	def put(body, pri=65536, delay=0, ttr=120)
+connection.rb:  	def yput(obj, pri=65536, delay=0, ttr=120)
+connection.rb:  	def reserve(timeout=nil)    
+connection.rb:  	def delete(id)
+connection.rb:  	def release(id, pri, delay)
+connection.rb:  	def bury(id, pri)
+
+connection.rb:  	def use(tube)
+connection.rb:  	def watch(tube)
+connection.rb:  	def ignore(tube)
+
+connection.rb:  	def peek_job(id)
+connection.rb:  	def peek_ready()
+connection.rb:  	def peek_delayed()
+connection.rb:  	def peek_buried()
+
+connection.rb:  	def stats()
+connection.rb:  	def job_stats(id)
+
+connection.rb:  	def stats_tube(tube)
+connection.rb:  	def list_tubes()
+connection.rb:  	def list_tube_used()
+connection.rb:  	def list_tubes_watched(cached=false)
+
+== Pool ==
+
+connection.rb:  	def initialize(addrs, 	default_tube=nil)
+connection.rb:  	def connect()
+connection.rb:  	def open_connections()
+connection.rb:  	def last_server
+connection.rb:  	def put(body, pri=65536, delay=0, ttr=120)
+connection.rb:  	def yput(obj, pri=65536, delay=0, ttr=120)
+connection.rb:  	def reserve(timeout=nil)
+connection.rb:  	def use(tube)
+connection.rb:  	def watch(tube)
+connection.rb:  	def ignore(tube)
+connection.rb:  	def raw_stats()
+connection.rb:  	def stats()
+connection.rb:  	def raw_stats_tube(tube)
+connection.rb:  	def stats_tube(tube)
+connection.rb:  	def list_tubes()
+connection.rb:  	def list_tube_used()
+connection.rb:  	def list_tubes_watched(*args)
+connection.rb:  	def remove(conn)
+connection.rb:  	def close
+connection.rb:  	def peek_ready()
+connection.rb:  	def peek_delayed()
+connection.rb:  	def peek_buried()
+connection.rb:  	def peek_job(id)
+connection.rb:  	def call_wrap(c, *args)
+connection.rb:  	def retry_wrap(*args)
+connection.rb:  	def send_to_each_conn_first_res(*args)
+connection.rb:  	def send_to_rand_conn(*args)
+connection.rb:  	def send_to_all_conns(*args)
+connection.rb:  	def pick_connection()
+connection.rb:  	def make_hash(pairs)
+connection.rb:  	def map_hash(h)
+connection.rb:  	def compact_hash(hash)
+connection.rb:  	def sum_hashes(hs)
+connection.rb:  	def combine_stats(k, a, b)
+
+
+== Job ==
+
+job.rb:  		def [](name)
+job.rb:  		def []=(name, val)
+job.rb:  		def ybody()
+job.rb:  		def initialize(conn, id, body)
+job.rb:  		def delete()
+job.rb:  		def put_back(pri=self.pri, delay=0, ttr=self.ttr)
+job.rb:  		def release(newpri=pri, delay=0)
+job.rb:  		def bury(newpri=pri)
+job.rb:  		def stats()
+job.rb:  		def timeouts() stats['timeouts'] end
+job.rb:  		def time_left() stats['time-left'] end
+job.rb:  		def age() stats['age'] end
+job.rb:  		def state() stats['state'] end
+job.rb:  		def delay() stats['delay'] end
+job.rb:  		def pri() stats['pri'] end
+job.rb:  		def ttr() stats['ttr'] end
+job.rb:  		def server()
+job.rb:  		def decay(d=([1, delay].max * 1.3).ceil)
+job.rb:  		def to_s
+job.rb:  		def inspect

data/spec/edamame_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Edamame" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'edamame'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

data/utils/god/README-god.textile

@@ -0,0 +1,54 @@
+Keep running:
+
+* Worker queue (beanstalkd)
+* Backing store (ttserver)
+* Request queue-fed Scrapers (list of scripts)
+* Feed/Periodic scrapers
+* Constant scrapers
+
+* http://god.rubyforge.org/
+* http://railscasts.com/episodes/130-monitoring-with-god
+
+    sudo gem install god
+    god -c config/mailit.god
+    god status
+    god terminate
+    god log mailit-workling
+    kill `cat log/workling.pid`
+
+* http://nubyonrails.com/articles/about-this-blog-beanstalk-messaging-queue
+** The "god.conf":http://pastie.textmate.org/private/ovgxu2ihoicli2ktrwtbew is
+   taken from there.
+
+h2. Beanstalkd
+
+*Usage*:
+
+    beanstalkd  --help
+    Use: beanstalkd [OPTIONS]
+
+    Options:
+     -d       detach
+     -l ADDR  listen on address (default is 0.0.0.0)
+     -p PORT  listen on port (default is 11300)
+     -u USER  become user and group
+     -z SIZE  set the maximum job size in bytes (default is 65535)
+     -v       show version information
+     -h       show this help
+
+
+h2. Tokyo Tyrant
+
+*Usage*:
+
+    ttserver --help
+    ttserver: the server of Tokyo Tyrant
+
+    usage:
+    ttserver [-host name] [-port num] [-thnum num] [-tout num] [-dmn]
+      [-pid path] [-kl] [-log path] [-ld|-le] [-ulog path]
+      [-ulim num] [-uas] [-sid num]
+      [-mhost name] [-mport num] [-rts path] [-rcc] [-skel name]
+      [-ext path] [-extpc name period] [-mask expr] [-unmask expr] [dbname]
+
+

data/utils/god/beanstalkd_god.rb

@@ -0,0 +1,34 @@
+class BeanstalkdGod < GodProcess
+  BeanstalkdGod::CONFIG_DEFAULTS = {
+    :listen_on      => '0.0.0.0',
+    :port           => 11300,
+    :user           => nil,
+    :max_job_size   => '65535',
+    :max_cpu_usage  => 50.percent,
+    :max_mem_usage  => 500.megabytes,
+    :monitor_group  => 'beanstalkds',
+    :beanstalkd_exe => '/usr/local/bin/beanstalkd',
+  }
+  def initialize *args
+    super *args
+    self.config = BeanstalkdGod::CONFIG_DEFAULTS.compact.merge(self.config)
+  end
+
+  def self.kind
+    :beanstalkd
+  end
+
+  def start_command
+    [
+      config[:beanstalkd_exe],
+      "-l #{config[:listen_on]}",
+      "-p #{config[:port]}",
+      "-z #{config[:max_job_size]}",
+      config[:user] ? "-u #{config[:user]}" : "",
+    ].flatten.compact.join(" ")
+  end
+
+  def self.are_you_there_god_its_me_beanstalkd *args
+    create *args
+  end
+end

data/utils/god/edamame.god

@@ -0,0 +1,30 @@
+$: << File.dirname(__FILE__)
+require 'god_process'
+require 'god_email'
+require 'beanstalkd_god'
+require 'tyrant_god'
+require 'sinatra_god'
+require 'god_site_config'
+
+EDAMAME_DB_DIR = '/data/distdb'
+
+#
+# For debugging:
+#
+#   sudo god -c /Users/flip/ics/edamame/utils/god/edamame.god -D
+#
+# (for production, use the etc/initc.d script in this directory)
+#
+# TODO: define an EdamameDirector that lets us name these collections.
+#
+THE_FAITHFUL = [
+  [BeanstalkdGod, { :port => 11210 }],
+  [TyrantGod,     { :port => 11212, :db_dirname => EDAMAME_DB_DIR, :db_name => 'flat_delay_queue.tct' }],
+  [TyrantGod,     { :port => 11213, :db_dirname => EDAMAME_DB_DIR, :db_name => 'fetched_urls.tch' }],
+  # [SinatraGod,    { :port => 11219, :app_dirname => File.dirname(__FILE__)+'/../../app/edamame_san' }],
+]
+
+THE_FAITHFUL.each do |klass, config|
+  proc = klass.create(config.merge :flapping_notify => 'default')
+  proc.mkdirs!
+end

data/utils/god/god-etc-init-dot-d-example

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# God
+#
+# chkconfig: - 85 15
+# description: start, stop, restart God (bet you feel powerful)
+#
+#
+# Make this go with
+#   chmod +x /etc/init.d/god
+#   /usr/sbin/update-rc.d god defaults
+
+RETVAL=0
+
+case "$1" in
+    start)
+      /usr/bin/god -P /var/run/god.pid -l /var/log/god.log
+      /usr/bin/god load /etc/god.conf
+      RETVAL=$?
+  ;;
+    stop)
+      kill `cat /var/run/god.pid`
+      RETVAL=$?
+  ;;
+    restart)
+      kill `cat /var/run/god.pid`
+      /usr/bin/god -P /var/run/god.pid -l /var/log/god.log
+      /usr/bin/god load /etc/god.conf
+      RETVAL=$?
+  ;;
+    status)
+      RETVAL=$?
+  ;;
+    *)
+      echo "Usage: god {start|stop|restart|status}"
+      exit 1
+  ;;
+esac
+
+exit $RETVAL

data/utils/god/god_email.rb

@@ -0,0 +1,45 @@
+module God
+  def self.setup_email config
+    God::Contacts::Email.message_settings = {
+      :from           => config[:username], }
+    God.contact(:email) do |c|
+      c.name          = config[:to_name]
+      c.email         = config[:to]
+      c.group         = config[:group] || 'default'
+    end
+    if config[:address] then delivery_by_smtp(config)
+    else                     delivery_by_gmail(config) end
+  end
+
+  #
+  # GMail
+  #
+  # http://millarian.com/programming/ruby-on-rails/monitoring-thin-using-god-with-google-apps-notifications/
+  def self.delivery_by_gmail config
+    require 'tlsmail'
+    Net::SMTP.enable_tls(OpenSSL::SSL::VERIFY_NONE)
+    God::Contacts::Email.server_settings = {
+      :address        => 'smtp.gmail.com',
+      :tls            => 'true',
+      :port           => 587,
+      :domain         => config[:email_domain],
+      :user_name      => config[:username],
+      :password       => config[:password],
+      :authentication => :plain
+    }
+  end
+
+  #
+  # SMTP email
+  #
+  def self.delivery_by_smtp config
+    God::Contacts::Email.server_settings = {
+      :address        => config[:address],
+      :port           => 25,
+      :domain         => config[:email_domain],
+      :user_name      => config[:username],
+      :password       => config[:password],
+      :authentication => :plain,
+    }
+  end
+end

data/utils/god/god_process.rb

@@ -0,0 +1,140 @@
+class GodProcess
+  CONFIG_DEFAULTS = {
+    :monitor_group      => nil,
+    :uid                => nil,
+    :gid                => nil,
+    :start_notify       => nil,
+    :restart_notify     => nil,
+    :flapping_notify    => nil,
+
+    :process_log_dir    => '/var/log/god',
+
+    :start_grace_time   => 20.seconds,
+    :restart_grace_time => nil,         # start_grace_time+2 if nil
+    :default_interval   => 5.minutes,
+    :start_interval     => 5.minutes,
+    :mem_usage_interval => 20.minutes,
+    :cpu_usage_interval => 20.minutes,
+  }
+
+  attr_accessor :config
+  def initialize config
+    self.config = CONFIG_DEFAULTS.compact.merge(config)
+  end
+
+  def setup
+    LOG.info config.inspect
+    God.watch do |watcher|
+      config_watcher  watcher
+      setup_start     watcher
+      setup_restart   watcher
+      setup_lifecycle watcher
+    end
+  end
+  def self.create config={}
+    proc = self.new config
+    proc.setup
+    proc
+  end
+
+  def handle
+    (config[:handle] || "#{self.class.kind}_#{config[:port]}").to_s
+  end
+
+  def process_log_file
+    File.join(config[:process_log_dir], handle+".log")
+  end
+
+
+  def mkdirs!
+    require 'fileutils'
+    FileUtils.mkdir_p File.dirname(process_log_file)
+  end
+
+  # command to start the daemon
+  def start_command
+    config[:start_command]
+  end
+  # command to stop the daemon
+  # return nil to have god daemonize the process
+  def stop_command
+    config[:stop_command]
+  end
+  # command to restart
+  # if stop_command is nil, it lets god daemonize the process
+  # otherwise, by default it runs stop_command, pauses for 1 second, then runs start_command
+  def restart_command
+    return unless stop_command
+    [stop_command, "sleep 1", start_command].join(" && ")
+  end
+
+  def config_watcher watcher
+    watcher.name             = self.handle
+    watcher.start            = start_command
+    watcher.stop             = stop_command            if stop_command
+    watcher.restart          = restart_command         if restart_command
+    watcher.group            = config[:monitor_group]  if config[:monitor_group]
+    watcher.uid              = config[:uid]            if config[:uid]
+    watcher.gid              = config[:gid]            if config[:gid]
+    watcher.interval         = config[:default_interval]
+    watcher.start_grace      = config[:start_grace_time]
+    watcher.restart_grace    = config[:restart_grace_time] || (config[:start_grace_time] + 2.seconds)
+    watcher.behavior(:clean_pid_file)
+  end
+
+  #
+  def setup_start watcher
+    watcher.start_if do |start|
+      start.condition(:process_running) do |c|
+        c.interval = config[:start_interval]
+        c.running  = false
+        c.notify     = config[:start_notify] if config[:start_notify]
+      end
+    end
+  end
+
+  #
+  def setup_restart watcher
+    watcher.restart_if do |restart|
+      restart.condition(:memory_usage) do |c|
+        c.interval   = config[:mem_usage_interval] if config[:mem_usage_interval]
+        c.above      = config[:max_mem_usage] || 150.megabytes
+        c.times      = [3, 5] # 3 out of 5 intervals
+        c.notify       = config[:restart_notify] if config[:restart_notify]
+      end
+      restart.condition(:cpu_usage) do |c|
+        c.interval   = config[:cpu_usage_interval] if config[:cpu_usage_interval]
+        c.above      = config[:max_cpu_usage] || 50.percent
+        c.times      = 5
+        c.notify       = config[:restart_notify] if config[:restart_notify]
+      end
+    end
+  end
+
+  # Define lifecycle
+  def setup_lifecycle watcher
+    watcher.lifecycle do |on|
+      on.condition(:flapping) do |c|
+        c.to_state     = [:start, :restart]
+        c.times        = 10
+        c.within       = 15.minute
+        c.transition   = :unmonitored
+        c.retry_in     = 60.minutes
+        c.retry_times  = 5
+        c.retry_within = 12.hours
+        c.notify       = config[:flapping_notify] if config[:flapping_notify]
+      end
+    end
+  end
+end
+
+class Hash
+  # remove all key-value pairs where the value is nil
+  def compact
+    reject{|key,val| val.nil? }
+  end
+  # Replace the hash with its compacted self
+  def compact!
+    replace(compact)
+  end
+end