mloughran-job_queue 0.0.3

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Martyn Loughran
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,58 @@
+JobQueue
+========
+
+`job_queue` allows you to use lots of message queues with exactly the same interface so you don't need to worry about which queue to pick :)
+
+This should get you started:
+
+    require 'rubygems'
+    require 'job_queue'
+
+Before you can do anything you must specify an adapter to use
+
+    JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+
+Jobs can then be simply added to the queue
+
+    JobQueue.put("flubble bubble")
+    
+In your workers you'll want to subscribe to a queue
+
+    JobQueue.subscribe do |job|
+      puts job
+    end
+
+This subscribe block takes care of waiting for the next job to arrive and the block is passed exactly what you passed in. If you want to exit the loop just throw :stop.
+
+    JobQueue.subscribe do |job|
+      # Wait - I changed my mind!
+      throw :stop
+    end
+
+What should you put on the queue
+--------------------------------
+
+You might love Ruby right now, but why lock yourself in? Often the kinds of things you use queues for are the kind of things you'll want to optimize. This is a good place to start:
+
+    JSON.generate({:some => "hash"})
+    JSON.parse(job)
+
+Can you show me a nice processing daemon?
+-----------------------------------------
+
+Yes. Just a minute...
+
+Adapters
+========
+
+Take your pick! Right now we have:
+
+Beanstalk
+---------
+<http://xph.us/software/beanstalkd/>
+
+AMQP
+----
+<http://github.com/tmm1/amqp/>
+
+You need to run all your code within an eventmachine loop to use AMQP.

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rubygems'
+require 'rake/gempackagetask'
+require 'rubygems/specification'
+require 'date'
+require 'spec/rake/spectask'
+
+GEM = "job_queue"
+GEM_VERSION = "0.0.3"
+AUTHOR = "Martyn Loughran"
+EMAIL = "me@mloughran.com"
+HOMEPAGE = "http://github.com/mloughran/job_queue/"
+SUMMARY = "JobQueue means you don't have to worry about your queue any more!"
+
+spec = Gem::Specification.new do |s|
+  s.name = GEM
+  s.version = GEM_VERSION
+  s.platform = Gem::Platform::RUBY
+  s.has_rdoc = true
+  s.extra_rdoc_files = ["README.markdown", "LICENSE"]
+  s.summary = SUMMARY
+  s.description = s.summary
+  s.author = AUTHOR
+  s.email = EMAIL
+  s.homepage = HOMEPAGE
+  
+  # Uncomment this to add a dependency
+  # s.add_dependency "foo"
+  
+  s.require_path = 'lib'
+  s.autorequire = GEM
+  s.files = %w(LICENSE README.markdown Rakefile) + Dir.glob("{lib,spec}/**/*")
+end
+
+task :default => :spec
+
+desc "Run specs"
+Spec::Rake::SpecTask.new do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = %w(-fs --color)
+end
+
+
+Rake::GemPackageTask.new(spec) do |pkg|
+  pkg.gem_spec = spec
+end
+
+desc "install the gem locally"
+task :install => [:package] do
+  sh %{sudo gem install pkg/#{GEM}-#{GEM_VERSION}}
+end
+
+desc "create a gemspec file"
+task :make_spec do
+  File.open("#{GEM}.gemspec", "w") do |file|
+    file.puts spec.to_ruby
+  end
+end

data/lib/job_queue/adapters/amqp_adapter.rb

@@ -0,0 +1,28 @@
+require 'mq'
+
+class JobQueue::AMQPAdapter
+  def initialize
+    amq = MQ.new
+    @exchange = amq.direct('photo', :durable => true)
+    @queue = amq.queue('photo_worker', :durable => true)
+    @queue.bind(@exchange)
+  end
+  
+  def put(string)
+    @queue.publish(string, :persistent => true)
+  end
+  
+  def subscribe(error_report, &block)
+    EM.add_periodic_timer(0) do
+      begin
+        @queue.pop do |header, body| 
+          next unless body
+          JobQueue.logger.info "AMQP received #{body}"
+          yield body
+        end
+      rescue => e
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/beanstalk_adapter.rb

@@ -0,0 +1,24 @@
+require 'beanstalk-client'
+
+class JobQueue::BeanstalkAdapter
+  def initialize
+    @beanstalk = Beanstalk::Pool.new(['localhost:11300'])
+  end
+  
+  def put(string)
+    @beanstalk.put(string)
+  end
+  
+  def subscribe(error_report, &block)
+    loop do
+      begin
+        job = @beanstalk.reserve
+        JobQueue.logger.info "Beanstalk received #{job.body}"
+        yield job.body
+        job.delete
+      rescue => e
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/test_adapter.rb

@@ -0,0 +1,20 @@
+class JobQueue::TestAdapter
+  def initialize
+    @queue = []
+  end
+  
+  def put(string)
+    @queue << string
+  end
+  
+  def subscribe(error_report, &block)
+    loop do
+      begin
+        sleep 0.1 if @queue.empty?
+        yield @queue.shift
+      rescue
+        error_report.call(job.body, e)
+      end
+    end
+  end
+end

data/lib/job_queue/adapters/verbose_adapter.rb

@@ -0,0 +1,19 @@
+# This isn't a queue at all, it just writes to standard output.
+# 
+# It might be useful for testing.
+# 
+class JobQueue::VerboseAdapter
+  def initialize
+    
+  end
+  
+  def put(string)
+    JobQueue.logger.debug "===== NEW JOB ADDED TO QUEUE ===="
+    JobQueue.logger.debug string
+    JobQueue.logger.debug "===== END OF MESSAGE ============"
+  end
+  
+  def subscribe(error_report, &block)
+    raise "Not implemented. Use a better adapter!!"
+  end
+end

data/lib/job_queue/job_queue.rb

@@ -0,0 +1,52 @@
+# JobQueue abstracts the task of adding work to a queue.
+# 
+# Beanstalk is fantastic, but maybe not "enterprise grade".
+# 
+# AMQP is fantastic, but it's bloody complex and has to run inside an
+# eventmachine loop.
+# 
+# Take your pick!
+# 
+# Before use, an adapter must be chosen:
+# 
+#   JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+# 
+# Jobs can then be simply added to the queue with
+# 
+#   JobQueue.put("flubble bubble")
+# 
+class JobQueue
+  class << self
+    attr_accessor :adapter
+    attr_accessor :logger
+    
+    def logger
+      @logger ||= begin
+        logger = Logger.new(STDOUT)
+        logger.level = Logger::WARN
+        logger.debug("Created logger")
+        logger
+      end
+    end
+  end
+  
+  def self.put(string)
+    adapter.put(string)
+  end
+  
+  def self.subscribe(error_report = nil, &block)
+    catch :stop do
+      error_report ||= Proc.new do |job, e|
+        JobQueue.logger.error \
+          "Job failed\n" \
+          "==========\n" \
+          "Job content: #{job.inspect}\n" \
+          "Exception: #{e.message}\n" \
+          "#{e.backtrace.join("\n")}\n" \
+          "\n"
+      end
+      
+      adapter.subscribe(error_report, &block)
+    end
+  end
+end

data/lib/job_queue.rb

@@ -0,0 +1,7 @@
+require 'logger'
+require 'job_queue/job_queue'
+
+JobQueue.autoload 'AMQPAdapter', 'job_queue/adapters/amqp_adapter'
+JobQueue.autoload 'BeanstalkAdapter', 'job_queue/adapters/beanstalk_adapter'
+JobQueue.autoload 'TestAdapter', 'job_queue/adapters/test_adapter'
+JobQueue.autoload 'VerboseAdapter', 'job_queue/adapters/verbose_adapter'

data/spec/amqp_adapter_spec.rb

@@ -0,0 +1,8 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::AMQPAdapter do
+  
+  it "should write onto queue and fetch stuff back off" do
+    pending "need to figure out a nice way of running rspec inside EM block"
+  end
+end

data/spec/beanstalk_adapter_spec.rb

@@ -0,0 +1,48 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::BeanstalkAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::BeanstalkAdapter.new
+  end
+  
+  it "should write onto queue and fetch stuff back off" do
+    JobQueue.put("hello")
+    
+    JobQueue.subscribe do |job|
+      @job = job
+      throw :stop
+    end
+    
+    @job.should == "hello"
+  end
+  
+  it "should output message if error raised in job" do
+    JobQueue.put("hello")
+    
+    JobQueue.logger.should_receive(:error).with(/Job failed\w*/)
+    
+    index = 0
+    JobQueue.subscribe do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+  
+  it "should use error_report block if supplied" do
+    JobQueue.put("hello")
+    
+    error_report = Proc.new do |job, e|
+      JobQueue.logger.error "Yikes that broke matey!"
+    end
+    
+    JobQueue.logger.should_receive(:error).with("Yikes that broke matey!")
+    
+    index = 0
+    JobQueue.subscribe(error_report) do |job|
+      index +=1
+      raise 'foo' if index == 1
+      throw :stop
+    end
+  end
+end

data/spec/job_queue_spec.rb

@@ -0,0 +1,5 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue do
+  
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+$TESTING=true
+$:.push File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'rubygems'
+require 'job_queue'

data/spec/test_adapter_spec.rb

@@ -0,0 +1,39 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::TestAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::TestAdapter.new
+  end
+  
+  it "should write onto queue and fetch stuff back off" do
+    JobQueue.put("hello")
+    
+    JobQueue.subscribe do |job|
+      @job = job
+      throw :stop
+    end
+    
+    @job.should == "hello"
+  end
+  
+  it "should pull items off in the order the were added" do
+    JobQueue.put("foo")
+    JobQueue.put("bar")
+    
+    retrieved_jobs = []
+    
+    begin
+      Timeout::timeout(0.5) do
+        JobQueue.subscribe do |job|
+          retrieved_jobs << job
+        end
+      end
+    rescue Timeout::Error
+      
+    end
+    
+    retrieved_jobs[0].should == "foo"
+    retrieved_jobs[1].should == "bar"
+    retrieved_jobs[2].should == nil
+  end
+end

data/spec/verbose_adapter_spec.rb

@@ -0,0 +1,15 @@
+require File.dirname(__FILE__) + '/spec_helper'
+
+describe JobQueue::VerboseAdapter do
+  before :all do
+    JobQueue.adapter = JobQueue::VerboseAdapter.new
+  end
+  
+  it "should write onto queue and output a very verbose message to stdout" do
+    JobQueue.logger.should_receive(:debug).with("===== NEW JOB ADDED TO QUEUE ====")
+    JobQueue.logger.should_receive(:debug).with("hello")
+    JobQueue.logger.should_receive(:debug).with("===== END OF MESSAGE ============")
+    
+    JobQueue.put("hello")
+  end
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification 
+name: mloughran-job_queue
+version: !ruby/object:Gem::Version 
+  version: 0.0.3
+platform: ruby
+authors: 
+- Martyn Loughran
+autorequire: job_queue
+bindir: bin
+cert_chain: []
+
+date: 2009-02-05 00:00:00 -08:00
+default_executable: 
+dependencies: []
+
+description: JobQueue means you don't have to worry about your queue any more!
+email: me@mloughran.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.markdown
+- LICENSE
+files: 
+- LICENSE
+- README.markdown
+- Rakefile
+- lib/job_queue
+- lib/job_queue/adapters
+- lib/job_queue/adapters/amqp_adapter.rb
+- lib/job_queue/adapters/beanstalk_adapter.rb
+- lib/job_queue/adapters/test_adapter.rb
+- lib/job_queue/adapters/verbose_adapter.rb
+- lib/job_queue/job_queue.rb
+- lib/job_queue.rb
+- spec/amqp_adapter_spec.rb
+- spec/beanstalk_adapter_spec.rb
+- spec/job_queue_spec.rb
+- spec/spec_helper.rb
+- spec/test_adapter_spec.rb
+- spec/verbose_adapter_spec.rb
+has_rdoc: true
+homepage: http://github.com/mloughran/job_queue/
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: JobQueue means you don't have to worry about your queue any more!
+test_files: []
+