psyho-stalker 0.7.0

data/README.md

@@ -0,0 +1,120 @@
+Stalker - a job queueing DSL for Beanstalk
+==========================================
+
+[Beanstalkd](http://kr.github.com/beanstalkd/) is a fast, lightweight queueing backend inspired by mmemcached.  The [Ruby Beanstalk client](http://beanstalk.rubyforge.org/) is a bit raw, however, so Stalker provides a thin wrapper to make job queueing from your Ruby app easy and fun.
+
+Queueing jobs
+-------------
+
+From anywhere in your app:
+
+    require 'stalker'
+
+    Stalker.enqueue('email.send', :to => 'joe@example.com')
+    Stalker.enqueue('post.cleanup.all')
+    Stalker.enqueue('post.cleanup', :id => post.id)
+
+Working jobs
+------------
+
+In a standalone file, typically jobs.rb or worker.rb:
+
+    require 'stalker'
+    include Stalker
+
+    job 'email.send' do |args|
+      Pony.send(:to => args['to'], :subject => "Hello there")
+    end
+
+    job 'post.cleanup.all' do |args|
+      Post.all.each do |post|
+        enqueue('post.cleanup', :id => post.all)
+      end
+    end
+
+    job 'post.cleanup' do |args|
+      Post.find(args['id']).cleanup
+    end
+
+Running
+-------
+
+First, make sure you have Beanstalkd installed and running:
+
+    $ sudo port install beanstalkd
+    $ beanstalkd
+
+Stalker:
+
+    $ sudo gem install stalker
+
+Now run a worker using the stalk binary:
+
+    $ stalk jobs.rb
+    Working 3 jobs: [ email.send post.cleanup.all post.cleanup ]
+    Working send.email (email=hello@example.com)
+    Finished send.email in 31ms
+
+Stalker will log to stdout as it starts working each job, and then again when the job finishes including the ellapsed time in milliseconds.
+
+Filter to a list of jobs you wish to run with an argument:
+
+    $ stalk jobs.rb post.cleanup.all,post.cleanup
+    Working 2 jobs: [ post.cleanup.all post.cleanup ]
+
+In a production environment you may run one or more high-priority workers (limited to short/urgent jobs) and any number of regular workers (working all jobs).  For example, two workers working just the email.send job, and four running all jobs:
+
+    $ for i in 1 2; do stalk jobs.rb email.send > log/urgent-worker.log 2>&1; end
+    $ for i in 1 2 3 4; do stalk jobs.rb > log/worker.log 2>&1; end
+
+Error Handling
+-------------
+
+If you include an `error` block in your jobs definition, that block will be invoked when a worker encounters an error. You might use this to report errors to an external monitoring service:
+
+    error do |job_name, args, e|
+      Exceptional.handle(e)
+    end
+
+Before filter
+-------------
+
+If you wish to run a block of code prior to any job:
+
+    before do |job|
+      puts "About to work #{job}"
+    end
+
+Tidbits
+-------
+
+* Jobs are serialized as JSON, so you should stick to strings, integers, arrays, and hashes as arguments to jobs.  e.g. don't pass full Ruby objects - use something like an ActiveRecord/MongoMapper/CouchRest id instead.
+* Because there are no class definitions associated with jobs, you can queue jobs from anywhere without needing to include your full app's environment.
+* If you need to change the location of your Beanstalk from the default (localhost:11300), set BEANSTALK_URL in your environment, e.g. export BEANSTALK_URL=beanstalk://example.com:11300/
+* The stalk binary is just for convenience, you can also run a worker with a straight Ruby command:
+    $ ruby -r jobs -e Stalker.work
+
+Running the tests
+-----------------
+
+If you wish to hack on Stalker, install these extra gems:
+
+    $ gem install contest mocha turn
+
+Run the tests:
+
+    $ turn
+
+Meta
+----
+
+Created by Adam Wiggins
+
+Patches from Jamie Cobbett, Scott Water, Keith Rarick, Mark McGranaghan, Sean Walberg
+
+Heavily inspired by [Minion](http://github.com/orionz/minion) by Orion Henry
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/adamwiggins/stalker
+

data/Rakefile

@@ -0,0 +1,19 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+	s.name = "psyho-stalker"
+	s.summary = "A job queueing and background workers system using Beanstalkd."
+	s.description = "A job queueing and background workers system using Beanstalkd.  Inspired by the Minion gem."
+	s.author = "Adam Wiggins"
+	s.email = "adam@heroku.com"
+	s.homepage = "http://github.com/adamwiggins/stalker"
+	s.executables = [ "stalk" ]
+	s.rubyforge_project = "stalker"
+
+	s.add_dependency 'beanstalk-client'
+	s.add_dependency 'json_pure'
+
+	s.files = FileList["[A-Z]*", "{bin,lib}/**/*"]
+end
+
+Jeweler::GemcutterTasks.new

data/VERSION

@@ -0,0 +1 @@
+0.7.0

data/bin/stalk

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+STDERR.sync = STDOUT.sync = true
+
+require File.expand_path('../../lib/stalker', __FILE__)
+include Stalker
+
+usage = "stalk <jobs.rb> [<job>[,<job>,..]]"
+file = ARGV.shift or abort usage
+jobs = ARGV.shift.split(',') rescue nil
+
+file = "./#{file}" unless file.match(/^[\/.]/)
+
+require file
+
+trap('INT') do
+	puts "\rExiting"
+	exit
+end
+
+Stalker.work jobs

data/examples/enqueue.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'stalker'
+
+Stalker.enqueue('send.email', :email => 'hello@example.com')
+Stalker.enqueue('cleanup.strays')

data/examples/jobs.rb

@@ -0,0 +1,11 @@
+job 'send.email' do |args|
+	log "Sending email to #{args['email']}"
+end
+
+job 'transform.image' do |args|
+	log "Image transform"
+end
+
+job 'cleanup.strays' do |args|
+	log "Cleaning up"
+end

data/lib/stalker.rb

@@ -0,0 +1,177 @@
+require 'beanstalk-client'
+require 'json'
+require 'uri'
+require 'timeout'
+
+module Stalker
+	extend self
+
+	def connect(url)
+		@@url = url
+		beanstalk
+	end
+
+	def enqueue(job, args={}, opts={})
+		pri   = opts[:pri]   || 65536
+		delay = opts[:delay] || 0
+		ttr   = opts[:ttr]   || 120
+		beanstalk.use job
+		beanstalk.put [ job, args ].to_json, pri, delay, ttr
+	rescue Beanstalk::NotConnected => e
+		failed_connection(e)
+	end
+
+	def job(j, &block)
+		@@handlers ||= {}
+		@@handlers[j] = block
+	end
+
+	def before(&block)
+		@@before_handlers ||= []
+		@@before_handlers << block
+	end
+
+	def error(&blk)
+		@@error_handler = blk
+	end
+
+	class NoJobsDefined < RuntimeError; end
+	class NoSuchJob < RuntimeError; end
+
+	def prep(jobs=nil)
+		raise NoJobsDefined unless defined?(@@handlers)
+		@@error_handler = nil unless defined?(@@error_handler)
+
+		jobs ||= all_jobs
+
+		jobs.each do |job|
+			raise(NoSuchJob, job) unless @@handlers[job]
+		end
+
+		log "Working #{jobs.size} jobs: [ #{jobs.join(' ')} ]"
+
+		jobs.each { |job| beanstalk.watch(job) }
+
+		beanstalk.list_tubes_watched.each do |server, tubes|
+			tubes.each { |tube| beanstalk.ignore(tube) unless jobs.include?(tube) }
+		end
+	rescue Beanstalk::NotConnected => e
+		failed_connection(e)
+	end
+
+	def work(jobs=nil)
+		prep(jobs)
+		loop { work_one_job }
+	end
+
+	class JobTimeout < RuntimeError; end
+
+	def work_one_job
+		job = beanstalk.reserve
+		name, args = JSON.parse job.body
+		log_job_begin(name, args)
+		handler = @@handlers[name]
+		raise(NoSuchJob, name) unless handler
+
+		begin
+			Timeout::timeout(job.ttr - 1) do
+				if defined? @@before_handlers and @@before_handlers.respond_to? :each
+					@@before_handlers.each do |block|
+						block.call(name)
+					end
+				end
+				handler.call(args)
+			end
+		rescue Timeout::Error
+			raise JobTimeout, "#{name} hit #{job.ttr-1}s timeout"
+		end
+
+		job.delete
+		log_job_end(name)
+	rescue Beanstalk::NotConnected => e
+		failed_connection(e)
+	rescue SystemExit
+		raise
+	rescue => e
+		log_error exception_message(e)
+		job.bury rescue nil
+		log_job_end(name, 'failed')
+		error_handler.call(name, args, e) if error_handler
+	end
+
+	def failed_connection(e)
+		log_error exception_message(e)
+		log_error "*** Failed connection to #{beanstalk_url}"
+		log_error "*** Check that beanstalkd is running (or set a different BEANSTALK_URL)"
+		exit 1
+	end
+
+	def log_job_begin(name, args)
+		args_flat = unless args.empty?
+			'(' + args.inject([]) do |accum, (key,value)|
+				accum << "#{key}=#{value}"
+			end.join(' ') + ')'
+		else
+			''
+		end
+
+		log [ "Working", name, args_flat ].join(' ')
+		@job_begun = Time.now
+	end
+
+	def log_job_end(name, failed=false)
+		ellapsed = Time.now - @job_begun
+		ms = (ellapsed.to_f * 1000).to_i
+		log "Finished #{name} in #{ms}ms #{failed ? ' (failed)' : ''}"
+	end
+
+	def log(msg)
+		puts msg
+	end
+
+	def log_error(msg)
+		STDERR.puts msg
+	end
+
+	def beanstalk
+		@@beanstalk ||= Beanstalk::Pool.new([ beanstalk_host_and_port ])
+	end
+
+	def beanstalk_url
+		return @@url if defined?(@@url) and @@url
+		ENV['BEANSTALK_URL'] || 'beanstalk://localhost/'
+	end
+
+	class BadURL < RuntimeError; end
+
+	def beanstalk_host_and_port
+		uri = URI.parse(beanstalk_url)
+		raise(BadURL, beanstalk_url) if uri.scheme != 'beanstalk'
+		return "#{uri.host}:#{uri.port || 11300}"
+	end
+
+	def exception_message(e)
+		msg = [ "Exception #{e.class} -> #{e.message}" ]
+
+		base = File.expand_path(Dir.pwd) + '/'
+		e.backtrace.each do |t|
+			msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+		end
+
+		msg.join("\n")
+	end
+
+	def all_jobs
+		@@handlers.keys
+	end
+
+	def error_handler
+		@@error_handler
+	end
+
+	def clear!
+		@@handlers = nil
+		@@before_handlers = nil
+		@@error_handler = nil
+	end
+end

data/test/stalker_test.rb

@@ -0,0 +1,101 @@
+require File.expand_path('../../lib/stalker', __FILE__)
+require 'contest'
+require 'mocha'
+
+module Stalker
+	def log(msg); end
+	def log_error(msg); end
+end
+
+class StalkerTest < Test::Unit::TestCase
+	setup do
+		Stalker.clear!
+		$result = -1
+		$handled = false
+	end
+
+	def with_an_error_handler
+		Stalker.error do |job_name, args, e|
+			$handled = e.class
+			$job_name = job_name
+			$job_args = args
+		end
+	end
+
+	test "enqueue and work a job" do
+		val = rand(999999)
+		Stalker.job('my.job') { |args| $result = args['val'] }
+		Stalker.enqueue('my.job', :val => val)
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal val, $result
+	end
+
+	test "invoke error handler when defined" do
+		with_an_error_handler
+		Stalker.job('my.job') { |args| fail }
+		Stalker.enqueue('my.job', :foo => 123)
+		Stalker.prep
+		Stalker.work_one_job
+		assert $handled
+		assert_equal 'my.job', $job_name
+		assert_equal({'foo' => 123}, $job_args)
+	end
+
+	test "continue working when error handler not defined" do
+		Stalker.job('my.job') { fail }
+		Stalker.enqueue('my.job')
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal false, $handled
+	end
+
+	test "exception raised one second before beanstalk ttr reached" do
+		with_an_error_handler
+		Stalker.job('my.job') { sleep(3); $handled = "didn't time out" }
+		Stalker.enqueue('my.job', {}, :ttr => 2)
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal Stalker::JobTimeout, $handled
+	end
+
+	test "before filter gets run first" do
+		Stalker.before { |name| $flag = "i_was_here" }
+		Stalker.job('my.job') { |args| $handled = ($flag == 'i_was_here') }
+		Stalker.enqueue('my.job')
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal true, $handled
+	end
+
+	test "before filter passes the name of the job" do
+		Stalker.before { |name| $jobname = name }
+		Stalker.job('my.job') { true }
+		Stalker.enqueue('my.job')
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal 'my.job', $jobname
+	end
+
+	test "before filter can pass an instance var" do
+		Stalker.before { |name| @foo = "hello" }
+		Stalker.job('my.job') { |args| $handled = (@foo == "hello") }
+		Stalker.enqueue('my.job')
+		Stalker.prep
+		Stalker.work_one_job
+		assert_equal true, $handled
+	end
+
+	test "before filter invokes error handler when defined" do
+		with_an_error_handler
+		Stalker.before { |name| fail }
+		Stalker.job('my.job') {	}
+		Stalker.enqueue('my.job', :foo => 123)
+		Stalker.prep
+		Stalker.work_one_job
+		assert $handled
+		assert_equal 'my.job', $job_name
+		assert_equal({'foo' => 123}, $job_args)
+	end
+
+end

metadata

@@ -0,0 +1,103 @@
+--- !ruby/object:Gem::Specification 
+name: psyho-stalker
+version: !ruby/object:Gem::Version 
+  hash: 3
+  prerelease: false
+  segments: 
+  - 0
+  - 7
+  - 0
+  version: 0.7.0
+platform: ruby
+authors: 
+- Adam Wiggins
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-02-16 00:00:00 +01:00
+default_executable: stalk
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: beanstalk-client
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json_pure
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: A job queueing and background workers system using Beanstalkd.  Inspired by the Minion gem.
+email: adam@heroku.com
+executables: 
+- stalk
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- README.md
+- Rakefile
+- VERSION
+- bin/stalk
+- lib/stalker.rb
+- examples/enqueue.rb
+- examples/jobs.rb
+- test/stalker_test.rb
+has_rdoc: true
+homepage: http://github.com/adamwiggins/stalker
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: stalker
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: A job queueing and background workers system using Beanstalkd.
+test_files: 
+- examples/enqueue.rb
+- examples/jobs.rb
+- test/stalker_test.rb