simplejob 0.1.0

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/LICENSE.txt

@@ -0,0 +1,23 @@
+The MIT License
+
+Copyright (c) 2011 Ken Pratt (ken@kenpratt.net)
+Copyright (c) 2011 Ruboss Technology Corp (peter@ruboss.com)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+Simple Job Queue
+================
+
+Painless job queueing, backed by AMQP.
+
+The design goals were:
+
+* Super simple interface
+* Robust jobs that never get lost
+* Jobs don't get marked complete until they are finished (they survive reboots)
+* One job queue per worker pool, with jobs mapped to that queue (for easy monitoring)
+* Jobs which throw exceptions are logged and removed, preventing infinite job loops
+
+Hello World
+-----------
+
+Start your [RabbitMQ](http://www.rabbitmq.com/) server (install one if necessary).
+
+<pre>
+producer.rb:
+
+require "simplejob"
+
+SimpleJob.send("hello")
+</pre>
+
+<pre>
+consumer.rb:
+
+require "simplejob"
+
+SimpleJob::Worker.start do
+  handle "hello" do |props|
+    puts "got it!"
+  end
+end
+</pre>
+
+<pre>
+$ ruby -rubygems consumer.rb
+</pre>
+
+<pre>
+$ ruby -rubygems producer.rb
+</pre>
+
+More examples
+-------------
+
+See the [examples directory](http://github.com/kenpratt/simple-job-queue/tree/master/examples).

data/Rakefile

@@ -0,0 +1,37 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "simplejob"
+  gem.homepage = "http://github.com/kenpratt/simplejob"
+  gem.license = "MIT"
+  gem.summary = "Painless job queueing."
+  gem.description = "A simple AMQP-backed job queuing system."
+  gem.email = "ken@kenpratt.net"
+  gem.authors = ["Ken Pratt"]
+  gem.add_dependency "activesupport", ">= 3.0.1"
+  gem.add_dependency "json", ">= 1.5.1"
+  gem.add_dependency "amqp", "0.7.1"
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/examples/basic/README.md

@@ -0,0 +1,10 @@
+Basic producer/consumer
+=======================
+
+Open a couple of terminal windows with consumers, and then run the producer:
+
+    ruby -rubygems consumer.rb
+    ruby -rubygems consumer.rb
+    ruby -rubygems producer.rb
+
+You should see the consumers pulling requests of the queue and processing them. Try killing and restarting one or both of them, or even restarting RabbitMQ. No jobs should be lost.

data/examples/basic/consumer.rb

@@ -0,0 +1,10 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob::Worker.start do
+  handle "a_task" do |props|
+    puts "got x: #{props[:x]}"
+    sleep 1
+  end
+end

data/examples/basic/producer.rb

@@ -0,0 +1,8 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+1.upto(20).each do |x|
+  puts "sending x: #{x}"
+  SimpleJob.send("a_task", :x => x)
+end

data/examples/multiple_jobs/README.md

@@ -0,0 +1,11 @@
+Multiple jobs mapped to different consumers
+===========================================
+
+Open a few terminal windows with various combinations of consumers, and then run the producer:
+
+    ruby -rubygems drink_consumer.rb
+    ruby -rubygems food_consumer.rb
+    ruby -rubygems food_consumer.rb
+    ruby -rubygems producer.rb
+
+You should see the consumers pulling requests of the queue and processing them. Try killing and restarting them, or even restarting RabbitMQ. No jobs should be lost.

data/examples/multiple_jobs/drink_consumer.rb

@@ -0,0 +1,17 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob::Worker.start(:queue_name => "drinks") do
+
+  handle "coffee" do |props|
+    puts "caffeine, here i come"
+    sleep 1
+  end
+
+  handle "milkshake" do |props|
+    puts "a #{props[:flavor]} milkshake, excellent!"
+    sleep 1
+  end
+
+end

data/examples/multiple_jobs/food_consumer.rb

@@ -0,0 +1,22 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob::Worker.start(:queue_name => "food") do
+
+  handle "pie" do |props|
+    puts "mmm, a #{props[:size]} #{props[:flavor]} pie, my favorite!"
+    sleep 3
+  end
+
+  handle "ice_cream" do |props|
+    puts "gross, #{props[:flavor]} ice cream"
+    sleep 1
+  end
+
+  handle "mousse" do |props|
+    puts "what's a mousse?"
+    sleep 0.1
+  end
+
+end

data/examples/multiple_jobs/producer.rb

@@ -0,0 +1,11 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob.send("coffee")
+SimpleJob.send("pie", :flavor => "apple", :size => "12in")
+SimpleJob.send("ice_cream", :flavor => "vanilla")
+SimpleJob.send("milkshake", :flavor => "chocolate")
+SimpleJob.send("mousse")
+SimpleJob.send("mousse")
+SimpleJob.send("mousse")

data/examples/wildcard_handlers/README.md

@@ -0,0 +1,10 @@
+Wildcard handlers
+=================
+
+Open the consumer and wiretap in terminal windows, and then run the producer:
+
+    ruby -rubygems consumer.rb
+    ruby -rubygems wiretap.rb
+    ruby -rubygems producer.rb
+
+You should see the consumer pulling requests of the queue and processing them, and the wiretap printing everything out. Try killing and restarting them, or even restarting RabbitMQ. No jobs or wiretapped jobs should be lost.

data/examples/wildcard_handlers/consumer.rb

@@ -0,0 +1,14 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob::Worker.start(:queue_name => "foo") do
+
+  handle "foo.*" do |props|
+    puts "normal foo: #{props[:topic]}"
+  end
+
+  handle "foo.#" do |props|
+    puts "catchall foo: #{props[:topic]}"
+  end
+end

data/examples/wildcard_handlers/producer.rb

@@ -0,0 +1,10 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob.send("foo.bar")
+SimpleJob.send("foo.baz")
+SimpleJob.send("foo.bar.baz")
+SimpleJob.send("foo.bar.baz.bang")
+SimpleJob.send("zap")
+SimpleJob.send("zap.whizz")

data/examples/wildcard_handlers/wiretap.rb

@@ -0,0 +1,11 @@
+$:.unshift(File.expand_path("../../../lib", __FILE__))
+
+require "simplejob"
+
+SimpleJob::Worker.start(:queue_name => "wiretap") do
+
+  handle "#" do |props|
+    puts "wiretapped: #{props[:topic]}"
+  end
+
+end

data/lib/simplejob.rb

@@ -0,0 +1,23 @@
+require "json"
+require "logger"
+
+require "simplejob/client"
+require "simplejob/worker"
+
+module SimpleJob
+
+  # Send a work request
+  def self.send(topic, props = {})
+    Client.start do
+      raise "Message properties should be a Hash" unless props.kind_of?(Hash)
+      log.info "[simplejob] New job: #{topic}, #{props.inspect}"
+      publish(topic, props.to_json)
+      stop
+    end
+  end
+
+end
+
+def log
+  Logger.new(STDOUT)
+end

data/lib/simplejob/client.rb

@@ -0,0 +1,50 @@
+require "amqp"
+
+module SimpleJob
+  DEFAULT_EXCHANGE_NAME = "simplejob"
+
+  class Client
+    def self.start(opts = {}, &proc)
+      instance = new
+      instance.start(opts, &proc)
+      instance
+    end
+
+    def start(opts, &proc)
+      client = self
+      AMQP.start(opts) do
+        Signal.trap("INT") { puts; stop }
+        Signal.trap("TERM") { puts; stop }
+
+        @channel = AMQP::Channel.new
+        @exchange = exchange(opts[:exchange_name] || DEFAULT_EXCHANGE_NAME)
+
+        instance_exec(opts, &proc)
+      end
+    end
+
+    def stop
+      AMQP.stop { EM.stop }
+    end
+
+    def exchange(name)
+      @channel.topic(name, :durable => true, :auto_delete => false)
+    end
+
+    def queue(name)
+      @channel.queue(name, :durable => true, :auto_delete => false)
+    end
+
+    def bind(queue, key)
+      queue.bind(@exchange, :key => key)
+    end
+
+    def publish(topic, message)
+      @exchange.publish(message, :routing_key => topic, :persistent => true)
+    end
+
+    def subscribe(queue, &proc)
+      queue.subscribe({ :ack => true }, &proc)
+    end
+  end
+end

data/lib/simplejob/worker.rb

@@ -0,0 +1,90 @@
+require "active_support/core_ext/hash"
+
+module SimpleJob
+  class Worker < Client
+    def start(opts, &proc)
+      super do
+        # only fetch one message at a time instead of being greedy
+        @channel.prefetch(1)
+
+        # create the queue
+        @queue = queue(opts[:queue_name] || "job_queue")
+
+        # defer to instance to set up handlers
+        instance_exec(&proc)
+
+        # start processing jobs
+        start_handler_loop
+      end
+    end
+
+    def handle(key, &proc)
+      # bind the message to the queue
+      bind(@queue, key)
+
+      # register the handler
+      install_handler(key, proc)
+    end
+
+    private
+
+    def start_handler_loop
+      subscribe(@queue) do |headers, body|
+        unless AMQP.closing?
+          puts headers.inspect
+          topic = headers.properties[:routing_key]
+
+          begin
+            start_time = Time.now
+            log.info "\n#{"="*80}\n#{topic.upcase} BEGIN -- #{start_time}\n#{"="*80}"
+
+            props = JSON.parse(body).with_indifferent_access
+            props[:topic] = topic
+            log.info "[worker] Props: #{props.inspect}"
+
+            if handler = find_handler(topic)
+              handler.call(props)
+            else
+              log.error "[worker] no handler installed for #{topic}"
+            end
+
+            end_time = Time.now
+            elapsed = ((end_time - start_time) * 10).round / 10.0
+            log.info "\n#{"="*80}\n#{topic.upcase} END -- #{end_time} (elapsed: #{elapsed}s)\n#{"="*80}\n"
+          rescue Exception => error
+            log.error "[worker] handle blew up for #{topic} with body:\n#{body}\n#{error}\n#{error.backtrace.join("\n")}"
+          end
+
+          headers.ack
+        end
+      end
+    end
+
+    def install_handler(key, proc)
+      @handlers ||= {}
+      @handlers[key] = { :block => proc, :re => key_to_regex(key) }
+
+      @handler_keys ||= []
+      @handler_keys << key
+    end
+
+    def find_handler(topic)
+      return nil unless @handlers
+
+      # check for direct hit
+      if handler = @handlers[topic]
+        handler[:block]
+
+      # otherwise, test regexes
+      else
+        if key = @handler_keys.detect {|k| @handlers[k][:re].match(topic) }
+          @handlers[key][:block]
+        end
+      end
+    end
+
+    def key_to_regex(key)
+      Regexp.compile("^" + key.gsub('*', '[^\.]+').gsub('#', '.*') + "$")
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+require 'shoulda'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'simplejob'
+
+class Test::Unit::TestCase
+end

data/test/test_simplejob.rb

@@ -0,0 +1,7 @@
+require 'helper'
+
+class TestSimplejob < Test::Unit::TestCase
+  should "probably rename this file and start testing for real" do
+    flunk "hey buddy, you should probably rename this file and start testing for real"
+  end
+end

metadata

@@ -0,0 +1,133 @@
+--- !ruby/object:Gem::Specification 
+name: simplejob
+version: !ruby/object:Gem::Version 
+  hash: 27
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ken Pratt
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-05-06 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: activesupport
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 3
+        - 0
+        - 1
+        version: 3.0.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 5
+        - 1
+        version: 1.5.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: amqp
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 0
+        - 7
+        - 1
+        version: 0.7.1
+  type: :runtime
+  version_requirements: *id003
+description: A simple AMQP-backed job queuing system.
+email: ken@kenpratt.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE.txt
+- README.md
+files: 
+- .document
+- LICENSE.txt
+- README.md
+- Rakefile
+- VERSION
+- examples/basic/README.md
+- examples/basic/consumer.rb
+- examples/basic/producer.rb
+- examples/multiple_jobs/README.md
+- examples/multiple_jobs/drink_consumer.rb
+- examples/multiple_jobs/food_consumer.rb
+- examples/multiple_jobs/producer.rb
+- examples/wildcard_handlers/README.md
+- examples/wildcard_handlers/consumer.rb
+- examples/wildcard_handlers/producer.rb
+- examples/wildcard_handlers/wiretap.rb
+- lib/simplejob.rb
+- lib/simplejob/client.rb
+- lib/simplejob/worker.rb
+- test/helper.rb
+- test/test_simplejob.rb
+homepage: http://github.com/kenpratt/simplejob
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.1
+signing_key: 
+specification_version: 3
+summary: Painless job queueing.
+test_files: []
+