quebert 0.0.6 → 0.0.8

data/Gemfile

@@ -4,10 +4,10 @@ gem 'json'
 gem 'beanstalk-client'
 
 group :test do
-  gem 'rspec'
+  gem 'rspec',                    '1.3.0',        :require => nil
   gem 'ZenTest'
   gem 'ruby-debug'
-  gem 'activerecord'
+  gem 'activerecord',             '2.3.5'
   gem 'sqlite3-ruby'
   gem 'autotest-fsevent'
 end

data/README.rdoc

@@ -2,6 +2,8 @@
 
 async_observer is great, but is dated and doesn't really support running jobs outside of the async_send idiom. Quebert is an attempt to mix how jobs are run in other popular worker queue frameworks, like resque and dj, with async_observer so that you can have it both ways.
 
+Quebert is a serious project. Its used in a production environment at Poll Everywhere to handle everything from SMS message processing to account downgrades.
+
 A worker queue framework designed around Beanstalk. Features include:
 
 = Features

data/Rakefile

@@ -31,8 +31,6 @@ Spec::Rake::SpecTask.new(:rcov) do |spec|
   spec.rcov = true
 end
 
-task :spec => :check_dependencies
-
 task :default => :spec
 
 require 'rake/rdoctask'

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.0.8

data/lib/quebert/async_sender/active_record.rb

@@ -4,7 +4,7 @@ module Quebert
     # I'm not sure if I want to do this or build serializers per type of object...
     module ActiveRecord
       class RecordJob < Job
-        def perform(record, meth, args)
+        def perform(record, meth, *args)
           record.send(meth, *args)
         end
       end
@@ -17,7 +17,7 @@ module Quebert
       module InstanceMethods
         # The meat of dealing with ActiveRecord instances.
         def async_send(meth, *args)
-          RecordJob.new(self, meth, args).enqueue
+          RecordJob.new(self, meth, *args).enqueue
         end
       end
     end

data/lib/quebert/async_sender/instance.rb

@@ -3,7 +3,7 @@ module Quebert
     # Perform jobs on instances of classes
     module Instance
       class InstanceJob < Job
-        def perform(klass, init_args, meth, args)
+        def perform(klass, init_args, meth, *args)
           Support.constantize(klass).new(init_args).send(meth, *args)
         end
       end
@@ -44,7 +44,7 @@ module Quebert
       end
       
       def async_send(meth, *args)
-        InstanceJob.new(self.class.name, @_init_args, meth, args).enqueue
+        InstanceJob.new(self.class.name, @_init_args, meth, *args).enqueue
       end
     end
   end

data/lib/quebert/async_sender/object.rb

@@ -3,7 +3,7 @@ module Quebert
     # Perform jobs on Object methods (not instances)
     module Object
       class ObjectJob < Job
-        def perform(const, meth, args)
+        def perform(const, meth, *args)
           Support.constantize(const).send(meth, *args)
         end
       end
@@ -14,7 +14,7 @@ module Quebert
       
       module ClassMethods
         def async_send(meth, *args)
-          ObjectJob.new(self.name, meth, args).enqueue
+          ObjectJob.new(self.name, meth, *args).enqueue
         end
       end
     end

data/lib/quebert/backend/beanstalk.rb

@@ -5,8 +5,8 @@ module Quebert
     
     # Manage jobs on a Beanstalk queue out of process
     class Beanstalk < Beanstalk::Pool
-      def put(job)
-        super job.to_json
+      def put(job, *args)
+        super job.to_json, *args
       end
       
       def reserve_with_controller

data/lib/quebert/backend/in_process.rb

@@ -2,7 +2,7 @@ module Quebert
   module Backend
     # Drops jobs on an array in-process.
     class InProcess < Array
-      def put(job)
+      def put(job, *args)
         unshift job.to_json
       end
       

data/lib/quebert/backend/sync.rb

@@ -4,7 +4,7 @@ module Quebert
     # or could be used as a fallback if other backends fail to initialize
     class Sync
       def put(job, *args)
-        Controller::Base.new(job).perform
+        Controller::Base.new(Job.from_json(job.to_json)).perform
       end
     end
   end

data/lib/quebert/controller/beanstalk.rb

@@ -6,7 +6,19 @@ module Quebert
       
       def initialize(beanstalk_job, queue)
         @beanstalk_job, @queue = beanstalk_job, queue
-        @job = Job.from_json(beanstalk_job.body)
+
+        begin
+          @job = Job.from_json(beanstalk_job.body)
+        rescue Job::Delete
+          beanstalk_job.delete
+        rescue Job::Release
+          beanstalk_job.release nil, @job.delay
+        rescue Job::Bury
+          beanstalk_job.bury
+        rescue Exception => e
+          beanstalk_job.bury
+          raise e  
+        end
       end
       
       def perform
@@ -17,7 +29,7 @@ module Quebert
         rescue Job::Delete
           beanstalk_job.delete
         rescue Job::Release
-          beanstalk_job.release
+          beanstalk_job.release nil, @job.delay
         rescue Job::Bury
           beanstalk_job.bury
         rescue Exception => e

data/lib/quebert/job.rb

@@ -3,7 +3,12 @@ require 'json'
 module Quebert
   class Job
     attr_reader :args
+    attr_accessor :priority, :delay, :ttr
     
+    DEFAULT_JOB_PRIORITY = 65536
+    DEFAULT_JOB_DELAY = 0
+    DEFAULT_JOB_TTR = 120
+
     NotImplemented = Class.new(StandardError)
     
     Action  = Class.new(Exception)
@@ -13,6 +18,23 @@ module Quebert
     Release = Class.new(Action)
     
     def initialize(*args)
+      opts = args.last.is_a?(::Hash) ? args.pop : nil
+      
+      @priority = DEFAULT_JOB_PRIORITY
+      @delay = DEFAULT_JOB_DELAY
+      @ttr = DEFAULT_JOB_TTR
+
+      if opts
+        beanstalk_opts = opts.delete(:beanstalk)
+        args << opts unless opts.empty?
+        
+        if beanstalk_opts
+          @priority = beanstalk_opts[:priority] if beanstalk_opts[:priority]
+          @delay = beanstalk_opts[:delay] if beanstalk_opts[:delay]
+          @ttr = beanstalk_opts[:ttr] if beanstalk_opts[:ttr]
+        end
+      end
+
       @args = args.dup.freeze
     end
     
@@ -26,7 +48,7 @@ module Quebert
     end
     
     def enqueue
-      self.class.backend.put self
+      self.class.backend.put self, @priority, @delay, @ttr
     end
     
     def to_json

data/lib/quebert/serializer.rb

@@ -6,13 +6,20 @@ module Quebert
       def self.serialize(job)
         {
           'job' => job.class.name,
-          'args' => serialize_args(job.args)
+          'args' => serialize_args(job.args),
+          'priority' => job.priority,
+          'delay' => job.delay,
+          'ttr' => job.ttr
         }
       end
       
       def self.deserialize(hash)
         hash = Support.stringify_keys(hash)
-        Support.constantize(hash['job']).new(*deserialize_args(hash['args']))
+        job = Support.constantize(hash['job']).new(*deserialize_args(hash['args']))
+        job.priority = hash['priority']
+        job.delay = hash['delay']
+        job.ttr = hash['ttr']
+        job
       end
       
     private
@@ -58,7 +65,7 @@ module Quebert
         hash = Support.stringify_keys(hash)
         model = Support.constantize(hash.delete('model'))
         if attrs = Support.stringify_keys(hash.delete('attributes'))
-          if id = hash.delete('id')
+          if id = attrs.delete('id')
             # This has been persisited, so just find it from the db
             model.find(id)
           else

data/lib/quebert/worker.rb

@@ -15,7 +15,8 @@ module Quebert
       logger.info "Worker pid##{Process.pid} started with #{backend.class.name} backend"
       while controller = backend.reserve do
         begin
-          log controller.job, "performing with args #{controller.job.args.inspect}"
+          log controller.job, "performing with args #{controller.job.args.inspect}."
+          log controller.job, "Priority: #{controller.beanstalk_job.pri}, Delay: #{controller.beanstalk_job.delay}, TTR: #{controller.beanstalk_job.ttr}" if controller.respond_to?(:beanstalk_job)
           controller.perform
           log controller.job, "complete"
         rescue Exception => e

data/quebert.gemspec

@@ -1,88 +1,86 @@
 # Generated by jeweler
 # DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{quebert}
-  s.version = "0.0.6"
+  s.version = "0.0.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brad Gessler"]
-  s.date = %q{2010-10-11}
+  s.date = %q{2011-09-16}
   s.default_executable = %q{quebert}
   s.description = %q{A worker queue framework built around beanstalkd}
   s.email = %q{brad@bradgessler.com}
   s.executables = ["quebert"]
   s.extra_rdoc_files = [
     "LICENSE",
-     "README.rdoc"
+    "README.rdoc"
   ]
   s.files = [
     ".document",
-     ".gitignore",
-     "Gemfile",
-     "LICENSE",
-     "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "bin/quebert",
-     "lib/quebert.rb",
-     "lib/quebert/async_sender.rb",
-     "lib/quebert/async_sender/active_record.rb",
-     "lib/quebert/async_sender/class.rb",
-     "lib/quebert/async_sender/instance.rb",
-     "lib/quebert/async_sender/object.rb",
-     "lib/quebert/backend.rb",
-     "lib/quebert/backend/beanstalk.rb",
-     "lib/quebert/backend/in_process.rb",
-     "lib/quebert/backend/sync.rb",
-     "lib/quebert/command_line_runner.rb",
-     "lib/quebert/configuration.rb",
-     "lib/quebert/controller.rb",
-     "lib/quebert/controller/base.rb",
-     "lib/quebert/controller/beanstalk.rb",
-     "lib/quebert/job.rb",
-     "lib/quebert/serializer.rb",
-     "lib/quebert/support.rb",
-     "lib/quebert/support/pid_file.rb",
-     "lib/quebert/support/registry.rb",
-     "lib/quebert/worker.rb",
-     "quebert.gemspec",
-     "spec/async_sender_spec.rb",
-     "spec/backend_spec.rb",
-     "spec/command_line_runner_spec.rb",
-     "spec/configuration_spec.rb",
-     "spec/consumer_spec.rb",
-     "spec/job_spec.rb",
-     "spec/quebert_spec.rb",
-     "spec/serializer_spec.rb",
-     "spec/spec.opts",
-     "spec/spec_helper.rb",
-     "spec/support/active_record.rb",
-     "spec/support/jobs.rb",
-     "spec/support_spec.rb",
-     "spec/worker_spec.rb"
+    "Gemfile",
+    "LICENSE",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "bin/quebert",
+    "lib/quebert.rb",
+    "lib/quebert/async_sender.rb",
+    "lib/quebert/async_sender/active_record.rb",
+    "lib/quebert/async_sender/class.rb",
+    "lib/quebert/async_sender/instance.rb",
+    "lib/quebert/async_sender/object.rb",
+    "lib/quebert/backend.rb",
+    "lib/quebert/backend/beanstalk.rb",
+    "lib/quebert/backend/in_process.rb",
+    "lib/quebert/backend/sync.rb",
+    "lib/quebert/command_line_runner.rb",
+    "lib/quebert/configuration.rb",
+    "lib/quebert/controller.rb",
+    "lib/quebert/controller/base.rb",
+    "lib/quebert/controller/beanstalk.rb",
+    "lib/quebert/job.rb",
+    "lib/quebert/serializer.rb",
+    "lib/quebert/support.rb",
+    "lib/quebert/support/pid_file.rb",
+    "lib/quebert/support/registry.rb",
+    "lib/quebert/worker.rb",
+    "quebert.gemspec",
+    "spec/async_sender_spec.rb",
+    "spec/backend_spec.rb",
+    "spec/command_line_runner_spec.rb",
+    "spec/configuration_spec.rb",
+    "spec/consumer_spec.rb",
+    "spec/job_spec.rb",
+    "spec/quebert_spec.rb",
+    "spec/serializer_spec.rb",
+    "spec/spec.opts",
+    "spec/spec_helper.rb",
+    "spec/support/active_record.rb",
+    "spec/support/jobs.rb",
+    "spec/support_spec.rb",
+    "spec/worker_spec.rb"
   ]
   s.homepage = %q{http://github.com/bradgessler/quebert}
-  s.rdoc_options = ["--charset=UTF-8"]
   s.require_paths = ["lib"]
   s.rubygems_version = %q{1.3.7}
   s.summary = %q{A worker queue framework built around beanstalkd}
   s.test_files = [
     "spec/async_sender_spec.rb",
-     "spec/backend_spec.rb",
-     "spec/command_line_runner_spec.rb",
-     "spec/configuration_spec.rb",
-     "spec/consumer_spec.rb",
-     "spec/job_spec.rb",
-     "spec/quebert_spec.rb",
-     "spec/serializer_spec.rb",
-     "spec/spec_helper.rb",
-     "spec/support/active_record.rb",
-     "spec/support/jobs.rb",
-     "spec/support_spec.rb",
-     "spec/worker_spec.rb"
+    "spec/backend_spec.rb",
+    "spec/command_line_runner_spec.rb",
+    "spec/configuration_spec.rb",
+    "spec/consumer_spec.rb",
+    "spec/job_spec.rb",
+    "spec/quebert_spec.rb",
+    "spec/serializer_spec.rb",
+    "spec/spec_helper.rb",
+    "spec/support/active_record.rb",
+    "spec/support/jobs.rb",
+    "spec/support_spec.rb",
+    "spec/worker_spec.rb"
   ]
 
   if s.respond_to? :specification_version then
@@ -90,15 +88,21 @@ Gem::Specification.new do |s|
     s.specification_version = 3
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<beanstalk-client>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_runtime_dependency(%q<json>, [">= 0"])
       s.add_runtime_dependency(%q<beanstalk-client>, [">= 0"])
     else
+      s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<beanstalk-client>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 1.2.9"])
       s.add_dependency(%q<json>, [">= 0"])
       s.add_dependency(%q<beanstalk-client>, [">= 0"])
     end
   else
+    s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<beanstalk-client>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 1.2.9"])
     s.add_dependency(%q<json>, [">= 0"])
     s.add_dependency(%q<beanstalk-client>, [">= 0"])

data/spec/async_sender_spec.rb

@@ -79,4 +79,10 @@ describe AsyncSender::ActiveRecord do
     @q.reserve.perform.should eql(email)
   end
   
+  it "should async_send and successfully serialize param object" do
+    user = User.new(:first_name => 'Brad')
+    user2 = User.new(:first_name => 'Steel')
+    user.async_send(:email, user2)
+    @q.reserve.perform.first_name.should eql('Steel')
+  end
 end

data/spec/consumer_spec.rb

@@ -38,6 +38,15 @@ describe Controller::Beanstalk do
     @q.peek_buried.should_not be_nil
   end
   
+  it "should bury an AR job if an exception occurs deserializing it" do
+    @user = User.new(:first_name => "John", :last_name => "Doe", :email => "jdoe@gmail.com")
+    @user.id = 1
+    @q.put Serializer::ActiveRecord.serialize(@user)
+    @q.peek_ready.should_not be_nil
+    lambda{ @q.reserve.perform }.should raise_exception
+    @q.peek_buried.should_not be_nil
+  end
+
   context "job actions" do
     it "should delete job" do
       @q.put DeleteJob.new

data/spec/job_spec.rb

@@ -55,6 +55,35 @@ describe Quebert::Job do
         Adder.new(1,2,3).enqueue
       }.should change(@q, :size).by(1)
     end
-  end
-  
+
+    context "beanstalk backend" do
+      before(:all) do
+        Quebert.serializers.register 'ActiveRecord::Base', Serializer::ActiveRecord
+
+        @q = Backend::Beanstalk.new('localhost:11300','quebert-test')
+
+        Quebert::AsyncSender::ActiveRecord::RecordJob.backend = @q
+        Quebert::AsyncSender::Object::ObjectJob.backend = @q
+
+        @q.drain!
+      end
+
+      it "should enqueue and honor beanstalk options" do
+        user = User.new(:first_name => "Steel")
+        user.async_send(:email, "somebody", nil, nil, :beanstalk => {:priority => 1, :delay => 2, :ttr => 300})
+        job = @q.reserve
+        job.beanstalk_job.pri.should eql(1)
+        job.beanstalk_job.delay.should eql(2)
+        job.beanstalk_job.ttr.should eql(300)
+      end
+
+      it "should enqueue and honor beanstalk options" do
+        User.async_send(:emailizer, "somebody", nil, nil, :beanstalk => {:priority => 1, :delay => 2, :ttr => 300})
+        job = @q.reserve
+        job.beanstalk_job.pri.should eql(1)
+        job.beanstalk_job.delay.should eql(2)
+        job.beanstalk_job.ttr.should eql(300)
+      end
+    end
+  end  
 end

data/spec/serializer_spec.rb

@@ -16,6 +16,7 @@ describe Serializer::ActiveRecord do
     it "should deserialize" do
       u = Serializer::ActiveRecord.deserialize(Serializer::ActiveRecord.serialize(@user))
       u.first_name.should eql('Tom')
+      u.id.should eql(@user.id)
     end
   end
   
@@ -40,7 +41,7 @@ end
 
 describe Serializer::Job do
   before(:all) do
-    @args = [100, User.new(:first_name => 'Brad')]
+    @args = [100, User.new(:first_name => 'Brad'), {:beanstalk => {:priority => 1, :delay => 2, :ttr => 300}}]
     @job = Job.new(*@args)
   end
   
@@ -50,11 +51,17 @@ describe Serializer::Job do
     h['args'][0]['payload'].should eql(100)
     h['args'][1]['payload'].should eql(Serializer::ActiveRecord.serialize(@args[1]))
     h['args'][1]['serializer'].should eql('Quebert::Serializer::ActiveRecord')
+    h['priority'].should eql(1)
+    h['delay'].should eql(2)
+    h['ttr'].should eql(300)
   end
   
   it "should deserialize job" do
     job = Serializer::Job.deserialize(Serializer::Job.serialize(@job))
     job.args[0].should eql(100)
     job.args[1].first_name.should eql('Brad')
+    job.delay.should eql(2)
+    job.priority.should eql(1)
+    job.ttr.should eql(300)
   end
 end

data/spec/support/active_record.rb

@@ -23,4 +23,8 @@ class User < ActiveRecord::Base
   def self.emailizer(address)
     address
   end
+
+  def email(address)
+    address
+  end
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: quebert
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 15
   prerelease: false
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 8
+  version: 0.0.8
 platform: ruby
 authors: 
 - Brad Gessler
@@ -15,13 +15,39 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-11 00:00:00 -07:00
+date: 2011-09-16 00:00:00 -07:00
 default_executable: quebert
 dependencies: 
 - !ruby/object:Gem::Dependency 
-  name: rspec
-  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: json
+  prerelease: false
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  name: beanstalk-client
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -33,11 +59,11 @@ dependencies:
         - 9
         version: 1.2.9
   type: :development
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: json
+  name: rspec
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -47,11 +73,11 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: beanstalk-client
+  name: json
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -61,7 +87,9 @@ dependencies:
         - 0
         version: "0"
   type: :runtime
-  version_requirements: *id003
+  name: beanstalk-client
+  prerelease: false
+  version_requirements: *id005
 description: A worker queue framework built around beanstalkd
 email: brad@bradgessler.com
 executables: 
@@ -73,7 +101,6 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - .document
-- .gitignore
 - Gemfile
 - LICENSE
 - README.rdoc
@@ -121,8 +148,8 @@ homepage: http://github.com/bradgessler/quebert
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 

data/.gitignore

@@ -1,22 +0,0 @@
-## MAC OS
-.DS_Store
-
-## TEXTMATE
-*.tmproj
-tmtags
-
-## EMACS
-*~
-\#*
-.\#*
-
-## VIM
-*.swp
-
-## PROJECT::GENERAL
-coverage
-rdoc
-pkg
-Gemfile.lock
-
-## PROJECT::SPECIFIC