strobemonkey-job_queue 0.0.5

data/README.markdown

@@ -0,0 +1,62 @@
+JobQueue
+========
+
+`job_queue` allows you to use lots of message queues with exactly the same interface so you don't need to worry about which queue to pick :)
+
+This should get you started:
+
+    require 'rubygems'
+    require 'job_queue'
+
+Before you can do anything you must specify an adapter to use
+
+    JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+
+Jobs can then be simply added to the queue
+
+    JobQueue.put("flubble bubble")
+    
+In your workers you'll want to subscribe to a queue
+
+    JobQueue.subscribe do |job|
+      puts job
+    end
+
+This subscribe block takes care of waiting for the next job to arrive and the block is passed exactly what you passed in. If you want to exit the loop just throw :stop.
+
+    JobQueue.subscribe do |job|
+      # Wait - I changed my mind!
+      throw :stop
+    end
+
+What should you put on the queue
+--------------------------------
+
+You might love Ruby right now, but why lock yourself in? Often the kinds of things you use queues for are the kind of things you'll want to optimize. This is a good place to start:
+
+    JSON.generate({:some => "hash"})
+    JSON.parse(job)
+
+Can you show me a nice processing daemon?
+-----------------------------------------
+
+Yes. Just a minute...
+
+Adapters
+========
+
+Take your pick! Right now we have:
+
+Beanstalk
+---------
+<http://xph.us/software/beanstalkd/>
+
+AMQP
+----
+<http://github.com/tmm1/amqp/>
+
+You need to run all your code within an eventmachine loop to use AMQP.
+
+Starling
+--------
+<http://github.com/starling/starling/>

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 4
+:major: 0
+:minor: 0

data/lib/job_queue.rb

@@ -0,0 +1,8 @@
+require 'logger'
+require 'job_queue/job_queue'
+
+JobQueue.autoload 'AMQPAdapter', 'job_queue/adapters/amqp_adapter'
+JobQueue.autoload 'BeanstalkAdapter', 'job_queue/adapters/beanstalk_adapter'
+JobQueue.autoload 'TestAdapter', 'job_queue/adapters/test_adapter'
+JobQueue.autoload 'VerboseAdapter', 'job_queue/adapters/verbose_adapter'
+JobQueue.autoload 'StarlingAdapter', 'job_queue/adapters/starling_adapter'

data/lib/job_queue/adapters/amqp_adapter.rb

@@ -0,0 +1,28 @@
+require 'mq'
+
+class JobQueue::AMQPAdapter
+  def initialize
+    amq = MQ.new
+    @exchange = amq.direct('photo', :durable => true)
+    @queue = amq.queue('photo_worker', :durable => true)
+    @queue.bind(@exchange)
+  end
+  
+  def put(string)
+    @queue.publish(string, :persistent => true)
+  end
+  
+  def subscribe(error_report, &block)
+    EM.add_periodic_timer(0) do
+      begin
+        @queue.pop do |header, body| 
+          next unless body
+          JobQueue.logger.info "AMQP received #{body}"
+          yield body
+        end
+      rescue => e
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/beanstalk_adapter.rb

@@ -0,0 +1,24 @@
+require 'beanstalk-client'
+
+class JobQueue::BeanstalkAdapter
+  def initialize
+    @beanstalk = Beanstalk::Pool.new(['localhost:11300'])
+  end
+  
+  def put(string)
+    @beanstalk.put(string)
+  end
+  
+  def subscribe(error_report, &block)
+    loop do
+      begin
+        job = @beanstalk.reserve
+        JobQueue.logger.info "Beanstalk received #{job.body}"
+        yield job.body
+        job.delete
+      rescue => e
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/starling_adapter.rb

@@ -0,0 +1,30 @@
+require 'memcache'
+require 'starling'
+
+class JobQueue::StarlingAdapter
+  def initialize(queue_name, server, port)
+    @queue_name = queue_name
+    @server = server
+    @port = port
+    @starling = Starling.new( @server + ':' + @port )
+  end
+
+  def put(string)
+    @starling.set(@queue_name, string)
+  end
+
+  def subscribe(error_report, &block)
+
+    loop do
+      begin
+        job = @starling.fetch(@queue_name)
+        JobQueue.logger.info "Starling received #{job}" 
+        yield job
+        sleep 0.25
+      rescue => e        
+        error_report.call(job, e)
+      end
+    end
+
+  end
+end

data/lib/job_queue/adapters/test_adapter.rb

@@ -0,0 +1,20 @@
+class JobQueue::TestAdapter
+  def initialize
+    @queue = []
+  end
+  
+  def put(string)
+    @queue << string
+  end
+  
+  def subscribe(error_report, &block)
+    loop do
+      begin
+        sleep 0.1 if @queue.empty?
+        yield @queue.shift
+      rescue
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/verbose_adapter.rb

@@ -0,0 +1,19 @@
+# This isn't a queue at all, it just writes to standard output.
+# 
+# It might be useful for testing.
+# 
+class JobQueue::VerboseAdapter
+  def initialize
+    
+  end
+  
+  def put(string)
+    JobQueue.logger.debug "===== NEW JOB ADDED TO QUEUE ===="
+    JobQueue.logger.debug string
+    JobQueue.logger.debug "===== END OF MESSAGE ============"
+  end
+  
+  def subscribe(error_report, &block)
+    raise "Not implemented. Use a better adapter!!"
+  end
+end

data/lib/job_queue/job_queue.rb

@@ -0,0 +1,52 @@
+# JobQueue abstracts the task of adding work to a queue.
+# 
+# Beanstalk is fantastic, but maybe not "enterprise grade".
+# 
+# AMQP is fantastic, but it's bloody complex and has to run inside an
+# eventmachine loop.
+# 
+# Take your pick!
+# 
+# Before use, an adapter must be chosen:
+# 
+#   JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+# 
+# Jobs can then be simply added to the queue with
+# 
+#   JobQueue.put("flubble bubble")
+# 
+class JobQueue
+  class << self
+    attr_accessor :adapter
+    attr_accessor :logger
+    
+    def logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::WARN
+        logger.debug("Created logger")
+        logger
+      end
+    end
+  end
+  
+  def self.put(string)
+    adapter.put(string)
+  end
+  
+  def self.subscribe(error_report = nil, &block)
+    catch :stop do
+      error_report ||= Proc.new do |job, e|
+        JobQueue.logger.error \
+          "Job failed\n" \
+          "==========\n" \
+          "Job content: #{job.inspect}\n" \
+          "Exception: #{e.message}\n" \
+          "#{e.backtrace.join("\n")}\n" \
+          "\n"
+      end
+      
+      adapter.subscribe(error_report, &block)
+    end
+  end
+end

data/spec/amqp_adapter_spec.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::AMQPAdapter do
+  
+  it "should write onto queue and fetch stuff back off" do
+    pending "need to figure out a nice way of running rspec inside EM block"
+  end
+end

data/spec/beanstalk_adapter_spec.rb

@@ -0,0 +1,48 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::BeanstalkAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+  end
+  
+  it "should write onto queue and fetch stuff back off" do
+    JobQueue.put("hello")
+    
+    JobQueue.subscribe do |job|
+      @job = job
+      throw :stop
+    end
+    
+    @job.should == "hello"
+  end
+  
+  it "should output message if error raised in job" do
+    JobQueue.put("hello")
+    
+    JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
+    
+    index = 0
+    JobQueue.subscribe do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+  
+  it "should use error_report block if supplied" do
+    JobQueue.put("hello")
+    
+    error_report = Proc.new do |job, e|
+      JobQueue.logger.error "Yikes that broke matey!"
+    end
+    
+    JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
+    
+    index = 0
+    JobQueue.subscribe(error_report) do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+end

data/spec/job_queue_spec.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue do
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'job_queue'

data/spec/starling_adapter_spec.rb

@@ -0,0 +1,56 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::StarlingAdapter do
+  
+  def fairly_unique_message
+    rand(0.1).to_s.split('.')[1]
+  end
+  
+  before :all do
+    JobQueue.adapter = JobQueue::StarlingAdapter.new( 'test_queue', 'localhost', '22122' )
+  end
+  
+  it "should write onto queue and fetch stuff back off" do    
+    message = fairly_unique_message
+    JobQueue.put(message)
+    
+    JobQueue.subscribe do |job|
+      @job = job
+      throw :stop
+    end
+    
+    @job.should == message
+  end
+  
+  it "should output message if error raised in job" do    
+    message = fairly_unique_message
+    JobQueue.put(message)
+    
+    JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
+
+    index = 0
+    JobQueue.subscribe do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+  
+  it "should use error_report block if supplied" do
+    message = fairly_unique_message
+    JobQueue.put(message)
+    
+    error_report = Proc.new do |job, e|
+      JobQueue.logger.error "Yikes that broke matey!"
+    end
+    
+    JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
+    
+    index = 0
+    JobQueue.subscribe(error_report) do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+end

data/spec/test_adapter_spec.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::TestAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::TestAdapter.new
+  end
+  
+  it "should write onto queue and fetch stuff back off" do
+    JobQueue.put("hello")
+    
+    JobQueue.subscribe do |job|
+      @job = job
+      throw :stop
+    end
+    
+    @job.should == "hello"
+  end
+  
+  it "should pull items off in the order the were added" do
+    JobQueue.put("foo")
+    JobQueue.put("bar")
+    
+    retrieved_jobs = []
+    
+    begin
+      Timeout::timeout(0.5) do
+        JobQueue.subscribe do |job|
+          retrieved_jobs << job
+        end
+      end
+    rescue Timeout::Error
+      
+    end
+    
+    retrieved_jobs[0].should == "foo"
+    retrieved_jobs[1].should == "bar"
+    retrieved_jobs[2].should == nil
+  end
+end

data/spec/verbose_adapter_spec.rb

@@ -0,0 +1,15 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::VerboseAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::VerboseAdapter.new
+  end
+  
+  it "should write onto queue and output a very verbose message to stdout" do
+    JobQueue.logger.should_receive(:debug).with("===== NEW JOB ADDED TO QUEUE ====")
+    JobQueue.logger.should_receive(:debug).with("hello")
+    JobQueue.logger.should_receive(:debug).with("===== END OF MESSAGE ============")
+    
+    JobQueue.put("hello")
+  end
+end

metadata

@@ -0,0 +1,71 @@
+--- !ruby/object:Gem::Specification 
+name: strobemonkey-job_queue
+version: !ruby/object:Gem::Version 
+  version: 0.0.5
+platform: ruby
+authors: 
+- Martyn Loughran
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-03-12 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: JobQueue means you don't have to worry about your queue any more!
+email: me@mloughran.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.markdown
+- VERSION.yml
+- lib/job_queue
+- lib/job_queue/adapters
+- lib/job_queue/adapters/amqp_adapter.rb
+- lib/job_queue/adapters/beanstalk_adapter.rb
+- lib/job_queue/adapters/test_adapter.rb
+- lib/job_queue/adapters/verbose_adapter.rb
+- lib/job_queue/adapters/starling_adapter.rb
+- lib/job_queue/job_queue.rb
+- lib/job_queue.rb
+- spec/amqp_adapter_spec.rb
+- spec/beanstalk_adapter_spec.rb
+- spec/job_queue_spec.rb
+- spec/spec_helper.rb
+- spec/test_adapter_spec.rb
+- spec/verbose_adapter_spec.rb
+- spec/starling_adapter_spec.rb
+has_rdoc: true
+homepage: http://github.com/mloughran/job_queue
+post_install_message: 
+rdoc_options: 
+- --inline-source
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: JobQueue means you don't have to worry about your queue any more!
+test_files: []
+