qu 0.1.0 → 0.1.1

data/Gemfile

@@ -2,7 +2,7 @@ source "http://rubygems.org"
 gemspec :name => 'qu'
 
 Dir['qu-*.gemspec'].each do |gemspec|
-  plugin = gemspec.scan(/qu-(.*)\.gemspec/).to_s
+  plugin = gemspec.scan(/qu-(.*)\.gemspec/).flatten.first
 
   group plugin do
     gemspec(:name => "qu-#{plugin}", :development_group => plugin)
@@ -10,8 +10,9 @@ Dir['qu-*.gemspec'].each do |gemspec|
 end
 
 group :test do
-  gem 'SystemTimer', :platform => :mri_18
-  gem 'ruby-debug',  :platform => :mri_18
+  gem 'SystemTimer',  :platform => :mri_18
+  gem 'ruby-debug',   :platform => :mri_18
+  gem 'ruby-debug19', :platform => :mri_19, :require => 'ruby-debug'
   gem 'rake'
   gem 'rspec', '~> 2.0'
   gem 'guard-rspec'

data/README.md

@@ -102,7 +102,8 @@ However, if you do need to customize it, you can by calling the `Qu.configure`:
 
 ``` ruby
 Qu.configure do |c|
-  c.connection = Redis::Namespace.new('myapp:qu', :redis => Redis.connect)
+  c.connection  = Redis::Namespace.new('myapp:qu', :redis => Redis.connect)
+  c.logger      = Logger.new('log/qu.log')
 end
 ```
 

data/Rakefile

@@ -17,7 +17,7 @@ task :release => :build do
   sh "git tag v#{Qu::VERSION}"
   sh "git push origin master"
   sh "git push origin v#{Qu::VERSION}"
-  sh "ls pkg/*.gem | xargs gem push"
+  sh "ls pkg/*.gem | xargs -n 1 gem push"
 end
 
 require 'rspec/core/rake_task'

data/lib/qu.rb

@@ -1,9 +1,11 @@
 require 'qu/version'
+require 'qu/logger'
 require 'qu/failure'
-require 'qu/job'
+require 'qu/payload'
 require 'qu/backend/base'
 
 require 'forwardable'
+require 'logger'
 
 module Qu
   autoload :Worker, 'qu/worker'
@@ -11,9 +13,9 @@ module Qu
   extend SingleForwardable
   extend self
 
-  attr_accessor :backend, :failure
+  attr_accessor :backend, :failure, :logger
 
-  def_delegators :backend, :enqueue, :length, :queues, :reserve, :clear, :connection=
+  def_delegators :backend, :length, :queues, :reserve, :clear, :connection=
 
   def backend
     @backend || raise("Qu backend not configured. Install one of the backend gems like qu-redis.")
@@ -22,6 +24,15 @@ module Qu
   def configure(&block)
     block.call(self)
   end
+
+  def enqueue(klass, *args)
+    backend.enqueue Payload.new(:klass => klass, :args => args)
+  end
+end
+
+Qu.configure do |c|
+  c.logger = Logger.new(STDOUT)
+  c.logger.level = Logger::INFO
 end
 
 require 'qu/railtie' if defined?(Rails)

data/lib/qu/backend/base.rb

@@ -3,6 +3,7 @@ require 'multi_json'
 module Qu
   module Backend
     class Base
+      include Logger
       attr_accessor :connection
 
     private

data/lib/qu/backend/spec.rb

@@ -1,3 +1,8 @@
+unless defined?(SystemTimer)
+  require 'timeout'
+  SystemTimer = Timeout
+end
+
 class SimpleJob
   def self.perform
   end
@@ -9,6 +14,7 @@ end
 
 shared_examples_for 'a backend' do
   let(:worker) { Qu::Worker.new('default') }
+  let(:payload) { Qu::Payload.new(:klass => SimpleJob) }
 
   before(:all) do
     Qu.backend = described_class.new
@@ -20,53 +26,58 @@ shared_examples_for 'a backend' do
   end
 
   describe 'enqueue' do
-    it 'should return a job id' do
-      subject.enqueue(SimpleJob).should be_instance_of(Qu::Job)
+    it 'should return a payload' do
+      subject.enqueue(payload).should be_instance_of(Qu::Payload)
+    end
+
+    it 'should set the payload id' do
+      subject.enqueue(payload)
+      payload.id.should_not be_nil
     end
 
     it 'should add a job to the queue' do
-      job = subject.enqueue(SimpleJob)
-      job.queue.should == 'default'
-      subject.length(job.queue).should == 1
+      subject.enqueue(payload)
+      payload.queue.should == 'default'
+      subject.length(payload.queue).should == 1
     end
 
     it 'should add queue to list of queues' do
       subject.queues.should == []
-      job = subject.enqueue(SimpleJob)
-      subject.queues.should == [job.queue]
+      subject.enqueue(payload)
+      subject.queues.should == [payload.queue]
     end
 
     it 'should assign a different job id for the same job enqueue multiple times' do
-      subject.enqueue(SimpleJob).id.should_not == subject.enqueue(SimpleJob).id
+      subject.enqueue(payload).id.should_not == subject.enqueue(payload).id
     end
   end
 
   describe 'length' do
     it 'should use the default queue by default' do
       subject.length.should == 0
-      subject.enqueue(SimpleJob)
+      subject.enqueue(payload)
       subject.length.should == 1
     end
   end
 
   describe 'clear' do
     it 'should clear jobs for given queue' do
-      job = subject.enqueue SimpleJob
-      subject.length(job.queue).should == 1
-      subject.clear(job.queue)
-      subject.length(job.queue).should == 0
-      subject.queues.should_not include(job.queue)
+      subject.enqueue payload
+      subject.length(payload.queue).should == 1
+      subject.clear(payload.queue)
+      subject.length(payload.queue).should == 0
+      subject.queues.should_not include(payload.queue)
     end
 
     it 'should not clear jobs for a different queue' do
-      job = subject.enqueue SimpleJob
+      subject.enqueue(payload)
       subject.clear('other')
-      subject.length(job.queue).should == 1
+      subject.length(payload.queue).should == 1
     end
 
     it 'should clear all queues without any args' do
-      subject.enqueue(SimpleJob).queue.should == 'default'
-      subject.enqueue(CustomQueue).queue.should == 'custom'
+      subject.enqueue(payload).queue.should == 'default'
+      subject.enqueue(Qu::Payload.new(:klass => CustomQueue)).queue.should == 'custom'
       subject.length('default').should == 1
       subject.length('custom').should == 1
       subject.clear
@@ -75,16 +86,16 @@ shared_examples_for 'a backend' do
     end
 
     it 'should clear failed queue without any args' do
-      job = subject.enqueue SimpleJob
-      subject.failed(job, Exception.new)
+      subject.enqueue(payload)
+      subject.failed(payload, Exception.new)
       subject.length('failed').should == 1
       subject.clear
       subject.length('failed').should == 0
     end
 
     it 'should not clear failed queue with specified queues' do
-      job = subject.enqueue SimpleJob
-      subject.failed(job, Exception.new)
+      subject.enqueue(payload)
+      subject.failed(payload, Exception.new)
       subject.length('failed').should == 1
       subject.clear('default')
       subject.length('failed').should == 1
@@ -93,26 +104,26 @@ shared_examples_for 'a backend' do
 
   describe 'reserve' do
     before do
-      @job = subject.enqueue SimpleJob
+      subject.enqueue(payload)
     end
 
     it 'should return next job' do
-      subject.reserve(worker).id.should == @job.id
+      subject.reserve(worker).id.should == payload.id
     end
 
     it 'should not return an already reserved job' do
-      another_job = subject.enqueue SimpleJob
+      subject.enqueue(payload)
       subject.reserve(worker).id.should_not == subject.reserve(worker).id
     end
 
     it 'should return next job in given queues' do
-      subject.enqueue SimpleJob
-      job = subject.enqueue CustomQueue
-      subject.enqueue SimpleJob
+      subject.enqueue(payload.dup)
+      custom = subject.enqueue(Qu::Payload.new(:klass => CustomQueue))
+      subject.enqueue(payload.dup)
 
       worker = Qu::Worker.new('custom', 'default')
 
-      subject.reserve(worker).id.should == job.id
+      subject.reserve(worker).id.should == custom.id
     end
 
     it 'should not return job from different queue' do
@@ -143,15 +154,15 @@ shared_examples_for 'a backend' do
   end
 
   describe 'failed' do
-    let(:job) { Qu::Job.new('1', SimpleJob, []) }
+    let(:payload) { Qu::Payload.new(:id => '1', :klass => SimpleJob) }
 
     it 'should add to failure queue' do
-      subject.failed(job, Exception.new)
+      subject.failed(payload, Exception.new)
       subject.length('failed').should == 1
     end
 
     it 'should not add failed queue to the list of queues' do
-      subject.failed(job, Exception.new)
+      subject.failed(payload, Exception.new)
       subject.queues.should_not include('failed')
     end
   end
@@ -164,45 +175,45 @@ shared_examples_for 'a backend' do
 
   describe 'release' do
     before do
-      subject.enqueue SimpleJob
+      subject.enqueue(payload)
     end
 
     it 'should add the job back on the queue' do
-      job = subject.reserve(worker)
-      subject.length(job.queue).should == 0
-      subject.release(job)
-      subject.length(job.queue).should == 1
+      subject.reserve(worker).id.should == payload.id
+      subject.length(payload.queue).should == 0
+      subject.release(payload)
+      subject.length(payload.queue).should == 1
     end
   end
 
   describe 'requeue' do
     context 'with a failed job' do
       before do
-        subject.enqueue(SimpleJob)
-        @job = subject.reserve(worker)
-        subject.failed(@job, Exception.new)
+        subject.enqueue(payload)
+        subject.reserve(worker).id.should == payload.id
+        subject.failed(payload, Exception.new)
       end
 
       it 'should add the job back on the queue' do
-        subject.length(@job.queue).should == 0
-        subject.requeue(@job.id)
-        subject.length(@job.queue).should == 1
-
-        job = subject.reserve(worker)
-        job.should be_instance_of(Qu::Job)
-        job.id.should == @job.id
-        job.klass.should == @job.klass
-        job.args.should == @job.args
+        subject.length(payload.queue).should == 0
+        subject.requeue(payload.id)
+        subject.length(payload.queue).should == 1
+
+        p = subject.reserve(worker)
+        p.should be_instance_of(Qu::Payload)
+        p.id.should == payload.id
+        p.klass.should == payload.klass
+        p.args.should == payload.args
       end
 
       it 'should remove the job from the failed jobs' do
         subject.length('failed').should == 1
-        subject.requeue(@job.id)
+        subject.requeue(payload.id)
         subject.length('failed').should == 0
       end
 
       it 'should return the job' do
-        subject.requeue(@job.id).id.should == @job.id
+        subject.requeue(payload.id).id.should == payload.id
       end
     end
 
@@ -214,8 +225,6 @@ shared_examples_for 'a backend' do
   end
 
   describe 'register_worker' do
-    let(:worker) { Qu::Worker.new('default') }
-
     it 'should add worker to array of workers' do
       subject.register_worker(worker)
       subject.workers.size.should == 1
@@ -234,10 +243,10 @@ shared_examples_for 'a backend' do
   end
 
   describe 'unregister_worker' do
-    before { subject.register_worker Qu::Worker.new('default') }
+    before { subject.register_worker(worker) }
 
     it 'should remove worker' do
-      subject.unregister_worker(worker.id)
+      subject.unregister_worker(worker)
       subject.workers.size.should == 0
     end
 
@@ -245,7 +254,7 @@ shared_examples_for 'a backend' do
       other_worker = Qu::Worker.new('other')
       subject.register_worker(other_worker)
       subject.workers.size.should == 2
-      subject.unregister_worker(other_worker.id)
+      subject.unregister_worker(other_worker)
       subject.workers.size.should == 1
       subject.workers.first.id.should == worker.id
     end

data/lib/qu/logger.rb

@@ -0,0 +1,20 @@
+module Qu
+  module Logger
+    def logger
+      Qu.logger
+    end
+
+    def log_exception(exception)
+      message = "\n#{exception.class} (#{exception.message}):\n  "
+      message << clean_backtrace(exception).join("\n  ") << "\n\n"
+      logger.fatal(message)
+    end
+
+    def clean_backtrace(exception)
+      defined?(Rails) && Rails.respond_to?(:backtrace_cleaner) ?
+        Rails.backtrace_cleaner.clean(exception.backtrace) :
+        exception.backtrace
+    end
+
+  end
+end

data/lib/qu/{job.rb → payload.rb}

@@ -1,11 +1,18 @@
+require 'ostruct'
+
 module Qu
-  class Job
-    attr_accessor :id, :klass, :args
+  class Payload < OpenStruct
+    include Logger
+
+    undef_method(:id) if method_defined?(:id)
 
-    def initialize(id, klass, args)
-      @id, @args = id, args
+    def initialize(options = {})
+      super
+      self.args ||= []
+    end
 
-      @klass = klass.is_a?(Class) ? klass : constantize(klass)
+    def klass
+      constantize(super)
     end
 
     def queue
@@ -16,16 +23,25 @@ module Qu
       klass.perform(*args)
       Qu.backend.completed(self)
     rescue Qu::Worker::Abort
+      logger.debug "Releasing job #{self}"
       Qu.backend.release(self)
       raise
     rescue Exception => e
+      logger.fatal "Job #{self} failed"
+      log_exception(e)
       Qu.failure.create(self, e) if Qu.failure
       Qu.backend.failed(self, e)
     end
 
+    def to_s
+      "#{id}:#{klass}:#{args.inspect}"
+    end
+
   protected
 
     def constantize(class_name)
+      return unless class_name
+      return class_name if class_name.is_a?(Class)
       constant = Object
       class_name.split('::').each do |name|
         constant = constant.const_get(name) || constant.const_missing(name)

data/lib/qu/railtie.rb

@@ -5,5 +5,9 @@ module Qu
     rake_tasks do
       load "qu/tasks.rb"
     end
+
+    initializer "qu.logger" do |app|
+      Qu.logger = Rails.logger
+    end
   end
 end

data/lib/qu/tasks.rb

@@ -5,3 +5,10 @@ namespace :qu do
     Qu::Worker.new(*queues).start
   end
 end
+
+# Convenience tasks compatibility
+task 'jobs:work'   => 'qu:work'
+task 'resque:work' => 'qu:work'
+
+# No-op task in case it doesn't already exist
+task :environment

data/lib/qu/version.rb

@@ -1,3 +1,3 @@
 module Qu
-  VERSION = "0.1.0"
+  VERSION = "0.1.1"
 end

data/lib/qu/worker.rb

@@ -1,5 +1,7 @@
 module Qu
   class Worker
+    include Logger
+
     attr_accessor :queues
 
     class Abort < Exception
@@ -22,23 +24,34 @@ module Qu
     end
 
     def handle_signals
+      logger.debug "Worker #{id} registering traps for INT and TERM signals"
       %W(INT TERM).each do |sig|
-        trap(sig) { raise Abort }
+        trap(sig) do
+          logger.info "Worker #{id} received #{sig}, shutting down"
+          raise Abort
+        end
       end
     end
 
     def work_off
+      logger.debug "Worker #{id} working of all jobs"
       while job = Qu.reserve(self, :block => false)
+        logger.debug "Worker #{id} reserved job #{job}"
         job.perform
+        logger.debug "Worker #{id} completed job #{job}"
       end
     end
 
     def work
+      logger.debug "Worker #{id} waiting for next job"
       job = Qu.reserve(self)
+      logger.debug "Worker #{id} reserved job #{job}"
       job.perform
+      logger.debug "Worker #{id} completed job #{job}"
     end
 
     def start
+      logger.warn "Worker #{id} starting"
       handle_signals
       Qu.backend.register_worker(self)
       loop { work }
@@ -46,10 +59,11 @@ module Qu
       # Ok, we'll shut down, but give us a sec
     ensure
       Qu.backend.unregister_worker(self)
+      logger.debug "Worker #{id} done"
     end
 
     def id
-      "#{hostname}:#{pid}:#{queues.join(',')}"
+      @id ||= "#{hostname}:#{pid}:#{queues.join(',')}"
     end
 
     def pid

data/qu.gemspec

@@ -2,7 +2,7 @@
 $:.push File.expand_path("../lib", __FILE__)
 require "qu/version"
 
-plugins = Dir['qu-*.gemspec'].map {|gemspec| gemspec.scan(/qu-(.*)\.gemspec/).to_s }.join('\|')
+plugins = Dir['qu-*.gemspec'].map {|gemspec| gemspec.scan(/qu-(.*)\.gemspec/).flatten.first }.join('\|')
 
 Gem::Specification.new do |s|
   s.name        = "qu"

data/spec/qu/{job_spec.rb → payload_spec.rb}

@@ -1,34 +1,42 @@
 require 'spec_helper'
 
-describe Qu::Job do
+describe Qu::Payload do
   class MyJob
     @queue = :custom
   end
 
+  it 'should default id to nil' do
+    Qu::Payload.new.id.should == nil
+  end
+
+  it 'should allow id to be set' do
+    Qu::Payload.new(:id => 5).id.should == 5
+  end
+
   describe 'queue' do
     it 'should default to "default"' do
-      Qu::Job.new('1', SimpleJob, []).queue.should == 'default'
+      Qu::Payload.new.queue.should == 'default'
     end
 
-    it 'should get queue from job instance variable' do
-      Qu::Job.new('1', MyJob, []).queue.should == 'custom'
+    it 'should get queue from instance variable' do
+      Qu::Payload.new(:klass => MyJob).queue.should == 'custom'
     end
   end
 
   describe 'klass' do
     it 'should constantize string' do
-      Qu::Job.new('1', 'MyJob', []).klass.should == MyJob
+      Qu::Payload.new(:klass => 'MyJob').klass.should == MyJob
     end
 
-    it 'should find namespaced jobs' do
-      Qu::Job.new('1', 'Qu::Job', []).klass.should == Qu::Job
+    it 'should find namespaced class' do
+      Qu::Payload.new(:klass => 'Qu::Payload').klass.should == Qu::Payload
     end
   end
 
   describe 'perform' do
-    subject { Qu::Job.new('1', SimpleJob, []) }
+    subject { Qu::Payload.new(:klass => SimpleJob) }
 
-    it 'should call .perform on job class with args' do
+    it 'should call .perform on class with args' do
       subject.args = ['a', 'b']
       SimpleJob.should_receive(:perform).with('a', 'b')
       subject.perform

data/spec/qu/worker_spec.rb

@@ -1,7 +1,7 @@
 require 'spec_helper'
 
 describe Qu::Worker do
-  let(:job) { Qu::Job.new('1', SimpleJob, []) }
+  let(:job) { Qu::Payload.new(:id => '1', :klass => SimpleJob) }
 
   describe 'queues' do
     it 'should use default if none specified' do

data/spec/qu_spec.rb

@@ -1,13 +1,25 @@
 require 'spec_helper'
 
 describe Qu do
-  %w(enqueue length queues reserve clear connection=).each do |method|
+  %w(length queues reserve clear connection=).each do |method|
     it "should delegate #{method} to backend" do
       Qu.backend.should_receive(method).with(:arg)
       Qu.send(method, :arg)
     end
   end
 
+  describe 'enqueue' do
+    it 'should call enqueue on backend with a payload' do
+      Qu.backend.should_receive(:enqueue) do |payload|
+        payload.should be_instance_of(Qu::Payload)
+        payload.klass.should == SimpleJob
+        payload.args.should == [9,8]
+      end
+
+      Qu.enqueue SimpleJob, 9, 8
+    end
+  end
+
   describe 'configure' do
     it 'should yield Qu' do
       Qu.configure do |c|

data/spec/spec_helper.rb

@@ -9,4 +9,6 @@ RSpec.configure do |config|
       :register_worker => nil, :unregister_worker => nil)
     Qu.failure = nil
   end
-end
+end
+
+Qu.logger = Logger.new('/dev/null')

metadata

@@ -1,47 +1,34 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: qu
-version: !ruby/object:Gem::Version 
-  hash: 27
+version: !ruby/object:Gem::Version
+  version: 0.1.1
   prerelease: 
-  segments: 
-  - 0
-  - 1
-  - 0
-  version: 0.1.0
 platform: ruby
-authors: 
+authors:
 - Brandon Keepers
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-09-23 00:00:00 -04:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-10-03 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: multi_json
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &70136826415300 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-description: ""
-email: 
+  prerelease: false
+  version_requirements: *70136826415300
+description: ''
+email:
 - brandon@opensoul.org
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - .gitignore
 - Gemfile
 - Guardfile
@@ -51,52 +38,43 @@ files:
 - lib/qu/backend/base.rb
 - lib/qu/backend/spec.rb
 - lib/qu/failure.rb
-- lib/qu/job.rb
+- lib/qu/logger.rb
+- lib/qu/payload.rb
 - lib/qu/railtie.rb
 - lib/qu/tasks.rb
 - lib/qu/version.rb
 - lib/qu/worker.rb
 - qu.gemspec
-- spec/qu/job_spec.rb
+- spec/qu/payload_spec.rb
 - spec/qu/worker_spec.rb
 - spec/qu_spec.rb
 - spec/spec_helper.rb
-has_rdoc: true
 homepage: http://github.com/bkeepers/qu
 licenses: []
-
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.6.1
+rubygems_version: 1.8.6
 signing_key: 
 specification_version: 3
-summary: ""
-test_files: 
-- spec/qu/job_spec.rb
+summary: ''
+test_files:
+- spec/qu/payload_spec.rb
 - spec/qu/worker_spec.rb
 - spec/qu_spec.rb
 - spec/spec_helper.rb