edamame 0.2.0

data/lib/methods.txt

@@ -0,0 +1,94 @@
+See also:
+
+  http://github.com/kr/beanstalkd/blob/master/doc/protocol.txt
+
+== Connection == 
+
+connection.rb:  	def initialize(addr, default_tube=nil)
+connection.rb:  	def connect
+connection.rb:  	def close
+
+connection.rb:  	def put(body, pri=65536, delay=0, ttr=120)
+connection.rb:  	def yput(obj, pri=65536, delay=0, ttr=120)
+connection.rb:  	def reserve(timeout=nil)    
+connection.rb:  	def delete(id)
+connection.rb:  	def release(id, pri, delay)
+connection.rb:  	def bury(id, pri)
+
+connection.rb:  	def use(tube)
+connection.rb:  	def watch(tube)
+connection.rb:  	def ignore(tube)
+
+connection.rb:  	def peek_job(id)
+connection.rb:  	def peek_ready()
+connection.rb:  	def peek_delayed()
+connection.rb:  	def peek_buried()
+
+connection.rb:  	def stats()
+connection.rb:  	def job_stats(id)
+
+connection.rb:  	def stats_tube(tube)
+connection.rb:  	def list_tubes()
+connection.rb:  	def list_tube_used()
+connection.rb:  	def list_tubes_watched(cached=false)
+
+== Pool ==
+
+connection.rb:  	def initialize(addrs, 	default_tube=nil)
+connection.rb:  	def connect()
+connection.rb:  	def open_connections()
+connection.rb:  	def last_server
+connection.rb:  	def put(body, pri=65536, delay=0, ttr=120)
+connection.rb:  	def yput(obj, pri=65536, delay=0, ttr=120)
+connection.rb:  	def reserve(timeout=nil)
+connection.rb:  	def use(tube)
+connection.rb:  	def watch(tube)
+connection.rb:  	def ignore(tube)
+connection.rb:  	def raw_stats()
+connection.rb:  	def stats()
+connection.rb:  	def raw_stats_tube(tube)
+connection.rb:  	def stats_tube(tube)
+connection.rb:  	def list_tubes()
+connection.rb:  	def list_tube_used()
+connection.rb:  	def list_tubes_watched(*args)
+connection.rb:  	def remove(conn)
+connection.rb:  	def close
+connection.rb:  	def peek_ready()
+connection.rb:  	def peek_delayed()
+connection.rb:  	def peek_buried()
+connection.rb:  	def peek_job(id)
+connection.rb:  	def call_wrap(c, *args)
+connection.rb:  	def retry_wrap(*args)
+connection.rb:  	def send_to_each_conn_first_res(*args)
+connection.rb:  	def send_to_rand_conn(*args)
+connection.rb:  	def send_to_all_conns(*args)
+connection.rb:  	def pick_connection()
+connection.rb:  	def make_hash(pairs)
+connection.rb:  	def map_hash(h)
+connection.rb:  	def compact_hash(hash)
+connection.rb:  	def sum_hashes(hs)
+connection.rb:  	def combine_stats(k, a, b)
+
+
+== Job ==
+
+job.rb:  		def [](name)
+job.rb:  		def []=(name, val)
+job.rb:  		def ybody()
+job.rb:  		def initialize(conn, id, body)
+job.rb:  		def delete()
+job.rb:  		def put_back(pri=self.pri, delay=0, ttr=self.ttr)
+job.rb:  		def release(newpri=pri, delay=0)
+job.rb:  		def bury(newpri=pri)
+job.rb:  		def stats()
+job.rb:  		def timeouts() stats['timeouts'] end
+job.rb:  		def time_left() stats['time-left'] end
+job.rb:  		def age() stats['age'] end
+job.rb:  		def state() stats['state'] end
+job.rb:  		def delay() stats['delay'] end
+job.rb:  		def pri() stats['pri'] end
+job.rb:  		def ttr() stats['ttr'] end
+job.rb:  		def server()
+job.rb:  		def decay(d=([1, delay].max * 1.3).ceil)
+job.rb:  		def to_s
+job.rb:  		def inspect

data/spec/edamame_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Edamame" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rubygems'
+require 'edamame'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+
+end

data/utils/god/edamame.god

@@ -0,0 +1,36 @@
+$: << File.dirname(__FILE__)+'/../../lib'
+$: << File.dirname(__FILE__)+'/../../../wukong/lib'
+$: << File.dirname(__FILE__)+'/../../../monkeyshines/lib'
+require 'edamame'
+require 'edamame/monitoring'
+
+#
+# For debugging:
+#
+#   sudo god -c /Users/flip/ics/edamame/utils/god/edamame.god -D
+#
+# (for production, use the etc/initc.d script in this directory)
+#
+# TODO: define an EdamameDirector that lets us name these collections.
+#
+
+GodProcess::GLOBAL_SITE_OPTIONS_FILES << ENV['HOME']+'/.edamame'
+
+# #
+# # Twitter
+# #
+# handle    = 'twitter'
+# God.process_group 11250, [
+#   # [ BeanstalkdGod, { :max_mem_usage => 100.megabytes } ],
+#   # [ TyrantGod,     { :db_name => handle+'-queue.tct' } ],
+#   # [ TyrantGod,     { :db_name => handle+'-scraped_at.tch' } ],
+#   [ SinatraGod,    { :thin_config_yml => '/slice/www/webshines/current/config.yml' } ],
+#   ]
+
+GodProcess.global_site_options[:process_groups].each do |handle, group_info|
+  group_info.each do |group, group_options|
+    klass = FactoryModule.get_class Kernel, group_options[:type]
+    klass.create(group_options)
+  end
+end
+

data/utils/god/edamame.yaml

@@ -0,0 +1,61 @@
+--- # -*- YAML -*-
+#
+# Save this file in your god dir, *then* change the settings below.
+# Make sure your version control system is set to ignore the file.
+#
+
+:god_notifiers:
+  :domain:              infochimps.org          #
+  :username:            robot@infochimps.org    # sender email username
+  :password:            xx_password_xx          # sender email password
+  :to:                  flip@infochimps.org     # recipient email address
+  :to_name:             Monitors                # recipient name
+
+# these apply to all processes
+:god_process:
+  :flapping_notify:     default
+
+:tyrant_god:
+  :db_dirname:          /data/distdb
+
+:sinatra_god:
+  :thin_config_yml:     '/slice/www/webshines/current/config.yml'
+  :server_exe:          '/usr/local/bin/thin'
+
+
+:process_groups:
+  :lastfm:
+    :queue_beanstalkd:
+      :type:                    :beanstalkd_god
+      :max_mem_usage:           100000000
+      :port:                    11250
+    :queue_db:
+      :type:                    :tyrant_god
+      :db_name:                 lastfm-queue.tct
+      :port:                    11251
+    :scraped_at_db:
+      :type:                    :tyrant_god
+      :db_name:                 lastfm-scraped_at.tch
+      :port:                    11252
+
+  :twitter_search:
+    :queue_beanstalkd:
+      :type:                    :beanstalkd_god
+      :max_mem_usage:           100000000
+      :port:                    11260
+    :queue_db:
+      :type:                    :tyrant_god
+      :db_name:                 twitter_search-queue.tct
+      :port:                    11261
+
+  :twitter_api:
+    :scraped_at_db:
+      :type:                    :tyrant_god
+      :db_name:                 twitter_api-scraped_at.tch
+      :port:                    11272
+    # :web_monitor:
+    #   :type:                  :sinatra_god
+    #   :thin_config_yml:       /slice/www/webshines/current/config.yml
+    #   :port:                  11275
+    #   :pid_file:              /var/run/god/thin.pid
+

data/utils/god/god-etc-init-dot-d-example

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# God
+#
+# chkconfig: - 85 15
+# description: start, stop, restart God (bet you feel powerful)
+#
+#
+# Make this go with
+#   chmod +x /etc/init.d/god
+#   /usr/sbin/update-rc.d god defaults
+
+RETVAL=0
+
+case "$1" in
+    start)
+      /usr/bin/god -P /var/run/god.pid -l /var/log/god.log
+      /usr/bin/god load /etc/god.conf
+      RETVAL=$?
+  ;;
+    stop)
+      kill `cat /var/run/god.pid`
+      RETVAL=$?
+  ;;
+    restart)
+      kill `cat /var/run/god.pid`
+      /usr/bin/god -P /var/run/god.pid -l /var/log/god.log
+      /usr/bin/god load /etc/god.conf
+      RETVAL=$?
+  ;;
+    status)
+      RETVAL=$?
+  ;;
+    *)
+      echo "Usage: god {start|stop|restart|status}"
+      exit 1
+  ;;
+esac
+
+exit $RETVAL

data/utils/god/god.conf

@@ -0,0 +1,22 @@
+# load in all god configs
+$: << '/home/flip/ics/edamame/lib'
+$: << '/home/flip/ics/wukong/lib'
+require 'yaml'
+require 'extlib'
+require 'wukong/extensions/hash'
+
+require "edamame/monitoring"
+GodProcess::GLOBAL_SITE_OPTIONS_FILES << '/slice/etc/god/.edamame'
+TyrantGod::DEFAULT_OPTIONS[:db_dirname] = '/data/distdb'
+
+p   GodProcess.global_site_options,
+    TyrantGod.site_options, TyrantGod.default_options.deep_merge(TyrantGod.site_options),
+    GodProcess.site_options
+
+#
+# Define email notifiers and attach one by default
+#
+God.setup_email GodProcess.global_site_options[:email]
+GodProcess::DEFAULT_OPTIONS[:flapping_notify] = 'default'
+
+God.load "/slice/etc/god/*.god"

data/utils/god/god_site_config.rb

@@ -0,0 +1,4 @@
+require 'yaml'
+SITE_OPTIONS_FILE = ENV['HOME']+'/.monkeyshines'
+SITE_OPTIONS = YAML.load(File.open(SITE_OPTIONS_FILE))
+God.setup_email SITE_OPTIONS[:email]

data/utils/god/wuclan.god

@@ -0,0 +1,36 @@
+#
+# For debugging:
+#
+#   sudo god -c /Users/flip/ics/edamame/utils/god/edamame.god -D
+#
+# (for production, use the etc/initc.d script in this directory)
+#
+# TODO: define an EdamameDirector that lets us name these collections.
+#
+
+#
+# Twitter Search
+#
+handle    = 'twitter_search'
+base_port = 11220
+BeanstalkdGod.create :port => base_port + 0, :max_mem_usage => 100.megabytes
+TyrantGod.create     :port => base_port + 1, :db_name => handle+'-queue.tct', :db_dirname => '/data/ripd/com.tw/com.twitter.search/distdb/20090928'
+
+#
+# Lastfm
+#
+handle    = 'lastfm'
+base_port = 11240
+BeanstalkdGod.create :port => base_port + 0, :max_mem_usage => 2.gigabytes
+TyrantGod.create     :port => base_port + 1, :db_name => handle+'-queue.tct',      :db_dirname => '/data/ripd/com.au/com.audioscrobbler.com/distdb/20090928'
+TyrantGod.create     :port => base_port + 2, :db_name => handle+'-scraped_at.tch', :db_dirname => '/data/ripd/com.au/com.audioscrobbler.com/distdb/20090928'
+
+# #
+# # Facebook
+# #
+# handle    = 'facebook'
+# base_port = 11250
+# BeanstalkdGod.create :port => base_port + 0, :max_mem_usage => 100.megabytes
+# TyrantGod.create     :port => base_port + 1, :db_name => handle+'-queue.tct'
+# TyrantGod.create     :port => base_port + 2, :db_name => handle+'-scraped_at.tch'
+# TyrantGod.create     :port => base_port + 3, :db_name => handle+'-registration.tch'

data/utils/simulation/Rescheduling Simulation.aliases

@@ -0,0 +1,3 @@
+[My Computer]
+My Computer = "192.168.69.129"
+

data/utils/simulation/Rescheduling Simulation.lvlps

@@ -0,0 +1,3 @@
+[ProjectWindow_Data]
+ProjectExplorer.ClassicPosition[String] = "52,33,416,383"
+

data/utils/simulation/Rescheduling Simulation.lvproj

@@ -0,0 +1,22 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<Project Type="Project" LVVersion="8608001">
+	<Item Name="My Computer" Type="My Computer">
+		<Property Name="server.app.propertiesEnabled" Type="Bool">true</Property>
+		<Property Name="server.control.propertiesEnabled" Type="Bool">true</Property>
+		<Property Name="server.tcp.enabled" Type="Bool">false</Property>
+		<Property Name="server.tcp.port" Type="Int">0</Property>
+		<Property Name="server.tcp.serviceName" Type="Str">My Computer/VI Server</Property>
+		<Property Name="server.tcp.serviceName.default" Type="Str">My Computer/VI Server</Property>
+		<Property Name="server.vi.callsEnabled" Type="Bool">true</Property>
+		<Property Name="server.vi.propertiesEnabled" Type="Bool">true</Property>
+		<Property Name="specify.custom.address" Type="Bool">false</Property>
+		<Item Name="Rescheduling.vi" Type="VI" URL="../Rescheduling.vi"/>
+		<Item Name="Weighted Average.vi" Type="VI" URL="../Weighted Average.vi"/>
+		<Item Name="Harmonic Average.vi" Type="VI" URL="../Harmonic Average.vi"/>
+		<Item Name="Add Percent Variation.vi" Type="VI" URL="../Add Percent Variation.vi"/>
+		<Item Name="Rescheduling with revisit.vi" Type="VI" URL="../Rescheduling with revisit.vi"/>
+		<Item Name="Rescheduling with revisit 2.vi" Type="VI" URL="../Rescheduling with revisit 2.vi"/>
+		<Item Name="Dependencies" Type="Dependencies"/>
+		<Item Name="Build Specifications" Type="Build"/>
+	</Item>
+</Project>

metadata

@@ -0,0 +1,147 @@
+--- !ruby/object:Gem::Specification 
+name: edamame
+version: !ruby/object:Gem::Version 
+  version: 0.2.0
+platform: ruby
+authors: 
+- Philip (flip) Kromer
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-12 00:00:00 -05:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: beanstalk-client
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: wukong
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: monkeyshines
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: "Edamame combines the Beanstalk priority queue with a Tokyo Tyrant database and God monitoring to produce a persistent distributed priority job queue system. \n\n\
+  Like beanstalk, it is fast, lightweight, distributed, priority queuing, reliable scheduling; it adds persistence, named jobs and job querying/enumeration. "
+email: flip@infochimps.org
+executables: 
+- edamame-nuke
+- edamame-ps
+- edamame-stats
+- edamame-sync
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.textile
+- README.textile
+files: 
+- LICENSE.textile
+- README.textile
+- app/edamame_san/config.ru
+- app/edamame_san/config.yml
+- app/edamame_san/edamame_san.rb
+- app/edamame_san/public/favicon.ico
+- app/edamame_san/public/images/edamame_logo.icns
+- app/edamame_san/public/images/edamame_logo.ico
+- app/edamame_san/public/images/edamame_logo.png
+- app/edamame_san/public/images/edamame_logo_2.icns
+- app/edamame_san/public/javascripts/application.js
+- app/edamame_san/public/javascripts/jquery/jquery-ui.js
+- app/edamame_san/public/javascripts/jquery/jquery.js
+- app/edamame_san/public/stylesheets/application.css
+- app/edamame_san/public/stylesheets/layout.css
+- app/edamame_san/views/layout.haml
+- app/edamame_san/views/load.haml
+- app/edamame_san/views/root.haml
+- bin/edamame-nuke
+- bin/edamame-ps
+- bin/edamame-stats
+- bin/edamame-sync
+- bin/edamame_util_opts.rb
+- bin/test_run.rb
+- lib/edamame.rb
+- lib/edamame/broker.rb
+- lib/edamame/job.rb
+- lib/edamame/monitoring.rb
+- lib/edamame/monitoring/README-god.textile
+- lib/edamame/monitoring/beanstalkd_god.rb
+- lib/edamame/monitoring/god_email.rb
+- lib/edamame/monitoring/god_process.rb
+- lib/edamame/monitoring/process_groups.rb
+- lib/edamame/monitoring/sinatra_god.rb
+- lib/edamame/monitoring/tyrant_god.rb
+- lib/edamame/persistent_queue.rb
+- lib/edamame/queue.rb
+- lib/edamame/queue/beanstalk.rb
+- lib/edamame/scheduling.rb
+- lib/edamame/store.rb
+- lib/edamame/store/base.rb
+- lib/edamame/store/tyrant_store.rb
+- lib/methods.txt
+- spec/edamame_spec.rb
+- spec/spec_helper.rb
+- utils/god/edamame.god
+- utils/god/edamame.yaml
+- utils/god/god-etc-init-dot-d-example
+- utils/god/god.conf
+- utils/god/god_site_config.rb
+- utils/god/wuclan.god
+- utils/simulation/Add Percent Variation.vi
+- utils/simulation/Harmonic Average.vi
+- utils/simulation/Rescheduling Simulation.aliases
+- utils/simulation/Rescheduling Simulation.lvlps
+- utils/simulation/Rescheduling Simulation.lvproj
+- utils/simulation/Rescheduling.vi
+- utils/simulation/Weighted Average.vi
+has_rdoc: true
+homepage: http://github.com/mrflip/edamame
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Beanstalk + Tokyo Tyrant = Edamame, a fast persistent distributed priority job queue.
+test_files: 
+- spec/edamame_spec.rb
+- spec/spec_helper.rb