stalker-be 0.1.0

data/README.md

@@ -0,0 +1,139 @@
+Stalker - a job queueing DSL for Beanstalk
+==========================================
+
+[Beanstalkd](http://kr.github.com/beanstalkd/) is a fast, lightweight queueing backend inspired by mmemcached. Stalker originally is a thin wrapper around the quite raw [Ruby Beanstalk client](http://beanstalk.rubyforge.org/), however, I updated it to use its successor [Beaneater](http://beanstalkd.github.com/beaneater/).
+
+Queueing jobs
+-------------
+
+From anywhere in your app:
+
+    require 'stalker'
+
+    Stalker.enqueue('email.send', :to => 'joe@example.com')
+    Stalker.enqueue('post.cleanup.all')
+    Stalker.enqueue('post.cleanup', :id => post.id)
+
+
+Job Status
+----------
+
+The current status of a job can be checked via
+
+    Stalker.status ‹job_id›
+
+The ‹job_id› is a String or Integer. It's returned from Stalker.enqueue.
+
+
+Working jobs
+------------
+
+In a standalone file, typically jobs.rb or worker.rb:
+
+    require 'stalker'
+    include Stalker
+
+    job 'email.send' do |args|
+      Pony.send(:to => args['to'], :subject => "Hello there")
+    end
+
+    job 'post.cleanup.all' do |args|
+      Post.all.each do |post|
+        enqueue('post.cleanup', :id => post.id)
+      end
+    end
+
+    job 'post.cleanup' do |args|
+      Post.find(args['id']).cleanup
+    end
+
+Running
+-------
+
+First, make sure you have Beanstalkd installed and running:
+
+    $ sudo port install beanstalkd
+    $ beanstalkd
+
+Stalker - with Bundler:
+
+    gem 'stalker', github: 'nkoehring/stalker'
+
+Stalker - with Bundler:
+    $ git clone git://github.com/nkoehring/stalker.git
+    $ cd stalker
+    $ gem build stalker.gemspec
+    $ gem install stalker-<version>.gem
+
+
+Now run a worker using the stalk binary:
+
+    $ stalk jobs.rb
+    Working 3 jobs: [ email.send post.cleanup.all post.cleanup ]
+    Working send.email (email=hello@example.com)
+    Finished send.email in 31ms
+
+Stalker will log to stdout as it starts working each job, and then again when the job finishes including the ellapsed time in milliseconds.
+
+Filter to a list of jobs you wish to run with an argument:
+
+    $ stalk jobs.rb post.cleanup.all,post.cleanup
+    Working 2 jobs: [ post.cleanup.all post.cleanup ]
+
+In a production environment you may run one or more high-priority workers (limited to short/urgent jobs) and any number of regular workers (working all jobs).  For example, two workers working just the email.send job, and four running all jobs:
+
+    $ for i in 1 2; do stalk jobs.rb email.send > log/urgent-worker.log 2>&1; done
+    $ for i in 1 2 3 4; do stalk jobs.rb > log/worker.log 2>&1; done
+
+
+Error Handling
+-------------
+
+If you include an `error` block in your jobs definition, that block will be invoked when a worker encounters an error. You might use this to report errors to an external monitoring service:
+
+    error do |e, job, args|
+      Exceptional.handle(e)
+    end
+
+Before filter
+-------------
+
+If you wish to run a block of code prior to any job:
+
+    before do |job|
+      puts "About to work #{job}"
+    end
+
+Tidbits
+-------
+
+* Jobs are serialized as JSON, so you should stick to strings, integers, arrays, and hashes as arguments to jobs.  e.g. don't pass full Ruby objects - use something like an ActiveRecord/MongoMapper/CouchRest id instead.
+* Because there are no class definitions associated with jobs, you can queue jobs from anywhere without needing to include your full app's environment.
+* If you need to change the location of your Beanstalk from the default (localhost:11300), set BEANSTALK_URL in your environment, e.g. export BEANSTALK_URL=beanstalk://example.com:11300/. You can specify multiple beanstalk servers, separated by whitespace or comma, e.g. export BEANSTALK_URL="beanstalk://b1.example.com:11300/, beanstalk://b2.example.com:11300/"
+* The stalk binary is just for convenience, you can also run a worker with a straight Ruby command:
+    $ ruby -r jobs -e Stalker.work
+
+Running the tests
+-----------------
+
+If you wish to hack on Stalker, install these extra gems:
+
+    $ gem install contest mocha
+
+Make sure you have a beanstalkd running, then run the tests:
+
+    $ ruby test/stalker_test.rb
+
+Meta
+----
+
+Created by [Adam Wiggins](https://github.com/adamwiggins)
+
+Patches from Jamie Cobbett, Scott Water, Keith Rarick, Mark McGranaghan, Sean Walberg, Adam Pohorecki, Han Kessels
+
+Heavily inspired by [Minion](https://github.com/orionz/minion) by Orion Henry
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+https://github.com/han/stalker
+

data/Rakefile

@@ -0,0 +1,19 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+	s.name = "stalker"
+	s.summary = "A job queueing and background workers system using Beanstalkd."
+	s.description = "A job queueing and background workers system using Beanstalkd.  Inspired by the Minion gem."
+	s.author = "Adam Wiggins"
+	s.email = "adam@heroku.com"
+	s.homepage = "https://github.com/han/stalker"
+	s.executables = [ "stalk" ]
+	s.rubyforge_project = "stalker"
+
+	s.add_dependency 'beanstalk-client'
+	s.add_dependency 'json_pure'
+
+	s.files = FileList["[A-Z]*", "{bin,lib}/**/*"]
+end
+
+Jeweler::GemcutterTasks.new

data/VERSION

@@ -0,0 +1 @@
+0.9.0

data/bin/stalk

@@ -0,0 +1,21 @@
+#!/usr/bin/env ruby
+
+STDERR.sync = STDOUT.sync = true
+
+require File.expand_path('../../lib/stalker', __FILE__)
+extend Stalker
+
+usage = "stalk <jobs.rb> [<job>[,<job>,..]]"
+file = ARGV.shift or abort usage
+jobs = ARGV.shift.split(',') rescue nil
+
+file = "./#{file}" unless file.match(/^[\/.]/)
+
+require file
+
+trap('INT') do
+	puts "\rExiting"
+	exit
+end
+
+Stalker.work jobs

data/examples/enqueue.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../lib'
+require 'stalker'
+
+Stalker.enqueue('send.email', :email => 'hello@example.com')
+Stalker.enqueue('cleanup.strays')

data/examples/jobs.rb

@@ -0,0 +1,11 @@
+job 'send.email' do |args|
+  log "Sending email to #{args['email']}"
+end
+
+job 'transform.image' do |args|
+  log "Image transform"
+end
+
+job 'cleanup.strays' do |args|
+  log "Cleaning up"
+end

data/lib/stalker.rb

@@ -0,0 +1,194 @@
+require 'beaneater'
+require 'json'
+require 'uri'
+require 'timeout'
+
+module Stalker
+  extend self
+
+  def connect(url)
+    @@url = url
+    beanstalk
+  end
+
+  def enqueue(job, args={}, opts={})
+    pri   = opts[:pri]   || 65536
+    delay = [0, opts[:delay].to_i].max  
+    ttr   = opts[:ttr]   || 120
+    tube  = beanstalk.tubes[job]
+    job = tube.put [ job, args ].to_json, pri:pri, delay:delay, ttr:ttr
+    job[:id]
+  rescue Beaneater::NotConnected => e
+    failed_connection(e)
+  end
+
+  def job(j, &block)
+    @@handlers ||= {}
+    @@handlers[j] = block
+  end
+
+  def status(job_id)
+    job = beanstalk.jobs[job_id]
+    job.stats unless job.nil?
+  end
+
+  def before(&block)
+    @@before_handlers ||= []
+    @@before_handlers << block
+  end
+
+  def error(&blk)
+    @@error_handler = blk
+  end
+
+  class NoJobsDefined < RuntimeError; end
+  class NoSuchJob < RuntimeError; end
+
+  def prep(jobs=nil)
+    raise NoJobsDefined unless defined?(@@handlers)
+    @@error_handler = nil unless defined?(@@error_handler)
+
+    jobs ||= all_jobs
+
+    jobs.each do |job|
+      raise(NoSuchJob, job) unless @@handlers[job]
+    end
+
+    log "Working #{jobs.size} jobs: [ #{jobs.join(' ')} ]"
+
+    jobs.each { |job| beanstalk.watch(job) }
+
+    beanstalk.tubes.watched.each do |tube|
+      beanstalk.ignore(tube) unless jobs.include?(tube)
+    end
+  rescue Beaneater::NotConnected => e
+    failed_connection(e)
+  end
+
+  def work(jobs=nil)
+    prep(jobs)
+    loop { work_one_job }
+  end
+
+  class JobTimeout < RuntimeError; end
+
+  def work_one_job
+    job = beanstalk.jobs.reserve
+    name, args = JSON.parse job.body
+    log_job_begin(name, args)
+    handler = @@handlers[name]
+    raise(NoSuchJob, name) unless handler
+
+    begin
+      Timeout::timeout(job.ttr - 1) do
+        if defined? @@before_handlers and @@before_handlers.respond_to? :each
+          @@before_handlers.each do |block|
+            block.call(name)
+          end
+        end
+        handler.call(args)
+      end
+    rescue Timeout::Error
+      raise JobTimeout, "#{name} hit #{job.ttr-1}s timeout"
+    end
+
+    job.delete
+    log_job_end(name)
+  rescue Beaneater::NotConnected => e
+    failed_connection(e)
+  rescue SystemExit
+    raise
+  rescue => e
+    log_error exception_message(e)
+    job.bury rescue nil
+		log_job_end(name, 'failed') if @job_begun
+    if error_handler
+      if error_handler.arity == 1
+        error_handler.call(e)
+      else
+        error_handler.call(e, name, args)
+      end
+    end
+  end
+
+  def failed_connection(e)
+    log_error exception_message(e)
+    log_error "*** Failed connection to #{beanstalk_url}"
+    log_error "*** Check that beanstalkd is running (or set a different BEANSTALK_URL)"
+    exit 1
+  end
+
+  def log_job_begin(name, args)
+    args_flat = unless args.empty?
+      '(' + args.inject([]) do |accum, (key,value)|
+        accum << "#{key}=#{value}"
+      end.join(' ') + ')'
+    else
+      ''
+    end
+
+    log [ "Working", name, args_flat ].join(' ')
+    @job_begun = Time.now
+  end
+
+  def log_job_end(name, failed=false)
+    ellapsed = Time.now - @job_begun
+    ms = (ellapsed.to_f * 1000).to_i
+    log "Finished #{name} in #{ms}ms #{failed ? ' (failed)' : ''}"
+  end
+
+  def log(msg)
+    puts msg
+  end
+
+  def log_error(msg)
+    STDERR.puts msg
+  end
+
+  def beanstalk
+    @@beanstalk ||= Beaneater::Pool.new(beanstalk_addresses)
+  end
+
+  def beanstalk_url
+    return @@url if defined?(@@url) and @@url
+    ENV['BEANSTALK_URL'] || 'beanstalk://localhost/'
+  end
+
+  class BadURL < RuntimeError; end
+
+  def beanstalk_addresses
+    uris = beanstalk_url.split(/[\s,]+/)
+    uris.map {|uri| beanstalk_host_and_port(uri)}
+  end
+
+  def beanstalk_host_and_port(uri_string)
+    uri = URI.parse(uri_string)
+    raise(BadURL, uri_string) if uri.scheme != 'beanstalk'
+    "#{uri.host}:#{uri.port || 11300}"
+  end
+
+  def exception_message(e)
+    msg = [ "Exception #{e.class} -> #{e.message}" ]
+
+    base = File.expand_path(Dir.pwd) + '/'
+    e.backtrace.each do |t|
+      msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+    end
+
+    msg.join("\n")
+  end
+
+  def all_jobs
+    @@handlers.keys
+  end
+
+  def error_handler
+    @@error_handler
+  end
+
+  def clear!
+    @@handlers = nil
+    @@before_handlers = nil
+    @@error_handler = nil
+  end
+end

data/test/stalker_test.rb

@@ -0,0 +1,124 @@
+require File.expand_path('../../lib/stalker', __FILE__)
+require 'contest'
+require 'mocha'
+
+module Stalker
+  def log(msg); end
+  def log_error(msg); end
+end
+
+class StalkerTest < Test::Unit::TestCase
+  setup do
+    Stalker.clear!
+    $result = -1
+    $handled = false
+  end
+
+  def with_an_error_handler
+    Stalker.error do |e, job_name, args|
+      $handled = e.class
+      $job_name = job_name
+      $job_args = args
+    end
+  end
+
+  test "enqueue and work a job" do
+    val = rand(999999)
+    Stalker.job('my.job') { |args| $result = args['val'] }
+    Stalker.enqueue('my.job', :val => val)
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal val, $result
+  end
+
+  test "invoke error handler when defined" do
+    with_an_error_handler
+    Stalker.job('my.job') { |args| fail }
+    Stalker.enqueue('my.job', :foo => 123)
+    Stalker.prep
+    Stalker.work_one_job
+    assert $handled
+    assert_equal 'my.job', $job_name
+    assert_equal({'foo' => 123}, $job_args)
+  end
+
+  test "should be compatible with legacy error handlers" do
+    exception = StandardError.new("Oh my, the job has failed!")
+    Stalker.error { |e| $handled = e }
+    Stalker.job('my.job') { |args| raise exception }
+    Stalker.enqueue('my.job', :foo => 123)
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal exception, $handled
+  end
+
+  test "continue working when error handler not defined" do
+    Stalker.job('my.job') { fail }
+    Stalker.enqueue('my.job')
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal false, $handled
+  end
+
+  test "exception raised one second before beanstalk ttr reached" do
+    with_an_error_handler
+    Stalker.job('my.job') { sleep(3); $handled = "didn't time out" }
+    Stalker.enqueue('my.job', {}, :ttr => 2)
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal Stalker::JobTimeout, $handled
+  end
+
+  test "before filter gets run first" do
+    Stalker.before { |name| $flag = "i_was_here" }
+    Stalker.job('my.job') { |args| $handled = ($flag == 'i_was_here') }
+    Stalker.enqueue('my.job')
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal true, $handled
+  end
+
+  test "before filter passes the name of the job" do
+    Stalker.before { |name| $jobname = name }
+    Stalker.job('my.job') { true }
+    Stalker.enqueue('my.job')
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal 'my.job', $jobname
+  end
+
+  test "before filter can pass an instance var" do
+    Stalker.before { |name| @foo = "hello" }
+    Stalker.job('my.job') { |args| $handled = (@foo == "hello") }
+    Stalker.enqueue('my.job')
+    Stalker.prep
+    Stalker.work_one_job
+    assert_equal true, $handled
+  end
+
+  test "before filter invokes error handler when defined" do
+    with_an_error_handler
+    Stalker.before { |name| fail }
+    Stalker.job('my.job') {  }
+    Stalker.enqueue('my.job', :foo => 123)
+    Stalker.prep
+    Stalker.work_one_job
+    assert $handled
+    assert_equal 'my.job', $job_name
+    assert_equal({'foo' => 123}, $job_args)
+  end
+  
+  test "parse BEANSTALK_URL" do
+    ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300"
+    assert_equal Stalker.beanstalk_addresses, ["localhost:12300"]
+    ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300/, beanstalk://localhost:12301/"
+    assert_equal Stalker.beanstalk_addresses, ["localhost:12300","localhost:12301"]
+    ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300   beanstalk://localhost:12301"
+    assert_equal Stalker.beanstalk_addresses, ["localhost:12300","localhost:12301"]
+    ENV['BEANSTALK_URL'] = "beanstalk://localhost:12300, http://localhost:12301"
+    assert_raise Stalker::BadURL do
+      Stalker.beanstalk_addresses 
+    end
+  end
+
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification
+name: stalker-be
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Norman Köhring
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: beaneater
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json_pure
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A job queueing and background workers system using Beanstalkd. Forked
+  from original stalker to use beaneater.
+email: n@koehr.in
+executables:
+- stalk
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- README.md
+- Rakefile
+- VERSION
+- bin/stalk
+- lib/stalker.rb
+- examples/enqueue.rb
+- examples/jobs.rb
+- test/stalker_test.rb
+homepage: https://github.com/nkoehring/stalker
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.25
+signing_key: 
+specification_version: 3
+summary: A job queueing and background workers system using Beanstalkd.
+test_files:
+- examples/enqueue.rb
+- examples/jobs.rb
+- test/stalker_test.rb