delayed_job 4.0.0.beta1 → 4.0.0.beta2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTQxN2VkZDVhM2YxY2RhNzE0NzRiN2ZlMDE0ZmQ3Y2I3ZjM5ODkyMA==
+  data.tar.gz: !binary |-
+    NjI3N2EyYTkzY2E2M2YyNmE2OWYxNTVhOWZkNWI5ZGViZWEzYjIzNQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MjhjOGJkN2YxNGM0NzI0ZDIxOTE0ZGVjNTk0OGZhZWQxZmY4YTkxYTczNWRk
+    ZWIxNTEzYWE4Mzc4OGE0Mjc2NzhhMDQ0NjdiZGVjMDc1YzEyMDUyOTFmYmRl
+    OTdmNzYyZDQ4YzJjZWVlOGUzNjczY2VlOTczYTc5Mjk1ZjhjZTE=
+  data.tar.gz: !binary |-
+    MDcxN2FkZDMxYWVhOTI2NjBmN2FmNzgyN2I0MTZkMmM3YmNjZWM4NTk2MzVl
+    ZDA2ZWQ1ZWE4YTE3OGI0ZDlmMTNjMGQ0ZDgyNTllNWM1OTczNzQxNWY3ZTFk
+    NjFkM2ZjMGEzOGYyMTQzNjNhMjg5ODRjNGEzNTExNGM2YjJiZGE=

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+4.0.0.beta2 - 2013-05-28
+========================
+* Rails 4 compatibility
+* Threaded startup script for faster multi-worker startup
+* YAML compatibility changes
+* Added jobs:check rake task
+
 4.0.0.beta1 - 2013-03-02
 ========================
 * Rails 4 compatibility

data/README.md

@@ -40,11 +40,15 @@ for other backends](http://wiki.github.com/collectiveidea/delayed_job/backends).
 
 If you plan to use delayed_job with Active Record, add `delayed_job_active_record` to your `Gemfile`.
 
-    gem 'delayed_job_active_record'
+```ruby
+gem 'delayed_job_active_record'
+```
 
 If you plan to use delayed_job with Mongoid, add `delayed_job_mongoid` to your `Gemfile`.
 
-    gem 'delayed_job_mongoid'
+```ruby
+gem 'delayed_job_mongoid'
+```
 
 Run `bundle install` to install the backend and delayed_job gems.
 
@@ -67,11 +71,13 @@ Queuing Jobs
 ============
 Call `.delay.method(params)` on any object and it will be processed in the background.
 
-    # without delayed_job
-    @user.activate!(@device)
+```ruby
+# without delayed_job
+@user.activate!(@device)
 
-    # with delayed_job
-    @user.delay.activate!(@device)
+# with delayed_job
+@user.delay.activate!(@device)
+```
 
 If a method should always be run in the background, you can call
 `#handle_asynchronously` after the method declaration:
@@ -123,6 +129,8 @@ class LongTasks
 end
 ```
 
+If you ever want to call a `handle_asynchronously`'d method without Delayed Job, for instance while debugging something at the console, just add `_without_delay` to the method name. For instance, if your original method was `foo`, then call `foo_without_delay`.
+
 Rails 3 Mailers
 ===============
 Due to how mailers are implemented in Rails 3, we had to do a little work around to get delayed_job to work.
@@ -180,6 +188,8 @@ You can then do the following:
     # or to run in the foreground
     RAILS_ENV=production script/delayed_job run --exit-on-complete
 
+**Rails 4:** *replace script/delayed_job with bin/delayed_job*
+
 Workers can be running on any computer, as long as they have access to the
 database and their clock is in sync. Keep in mind that each worker will check
 the database at least every 5 seconds.
@@ -193,6 +203,21 @@ Work off queues by setting the `QUEUE` or `QUEUES` environment variable.
 
     QUEUE=tracking rake jobs:work
     QUEUES=mailers,tasks rake jobs:work
+    
+Restarting delayed_job
+======================
+
+The following syntax will restart delayed jobs:
+
+    RAILS_ENV=production script/delayed_job restart
+
+To restart multiple delayed_job workers:
+
+    RAILS_ENV=production script/delayed_job -n2 restart
+    
+**Rails 4:** *replace script/delayed_job with bin/delayed_job*
+
+
 
 Custom Jobs
 ===========
@@ -207,6 +232,19 @@ end
 
 Delayed::Job.enqueue NewsletterJob.new('lorem ipsum...', Customers.find(:all).collect(&:email))
 ```
+To set a per-job max attempts that overrides the Delayed::Worker.max_attempts you can define a max_attempts method on the job
+```ruby
+class NewsletterJob < Struct.new(:text, :emails)
+  def perform
+    emails.each { |e| NewsletterMailer.deliver_text_to_email(text, e) }
+  end
+  
+  def max_attempts
+    return 3
+  end
+end
+````
+
 
 Hooks
 =====

data/contrib/delayed_job_rails_4.monitrc

@@ -0,0 +1,14 @@
+# an example Monit configuration file for delayed_job
+# See: http://stackoverflow.com/questions/1226302/how-to-monitor-delayedjob-with-monit/1285611
+#
+# To use:
+# 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc
+# 2. replace {app_name} as appropriate
+# 3. add this to your /etc/monit/monitrc
+#
+#   include /var/www/apps/{app_name}/shared/delayed_job.monitrc
+
+check process delayed_job
+  with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.pid
+  start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start"
+  stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop"

data/contrib/delayed_job_rails_4_multiple.monitrc

@@ -0,0 +1,34 @@
+# an example Monit configuration file for delayed_job running multiple processes
+#
+# To use:
+# 1. copy to /var/www/apps/{app_name}/shared/delayed_job.monitrc
+# 2. replace {app_name} as appropriate
+#    you might also need to change the program strings to
+#           "/bin/su - {username} -c '/usr/bin/env ...'"
+#    to load your shell environment.
+#
+# 3. add this to your /etc/monit/monitrc
+#
+#   include /var/www/apps/{app_name}/shared/delayed_job.monitrc
+#
+# The processes are grouped so that monit can act on them as a whole, e.g.
+#
+#   monit -g delayed_job restart
+
+check process delayed_job_0
+  with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.0.pid
+  start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 0"
+  stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 0"
+  group delayed_job
+
+check process delayed_job_1
+  with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.1.pid
+  start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 1"
+  stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 1"
+  group delayed_job
+
+check process delayed_job_2
+  with pidfile /var/www/apps/{app_name}/shared/pids/delayed_job.2.pid
+  start program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job start -i 2"
+  stop program = "/usr/bin/env RAILS_ENV=production /var/www/apps/{app_name}/current/bin/delayed_job stop -i 2"
+  group delayed_job

data/delayed_job.gemspec

@@ -13,5 +13,5 @@ Gem::Specification.new do |spec|
   spec.require_paths  = ['lib']
   spec.summary        = 'Database-backed asynchronous priority queue system -- Extracted from Shopify'
   spec.test_files     = Dir.glob('spec/**/*')
-  spec.version        = '4.0.0.beta1'
+  spec.version        = '4.0.0.beta2'
 end

data/lib/delayed/command.rb

@@ -78,10 +78,14 @@ module Delayed
         process_name = "delayed_job.#{@options[:identifier]}"
         run_process(process_name, dir)
       else
+        threads = []
         worker_count.times do |worker_index|
-          process_name = worker_count == 1 ? "delayed_job" : "delayed_job.#{worker_index}"
-          run_process(process_name, dir)
+          threads << Thread.start do
+            process_name = worker_count == 1 ? "delayed_job" : "delayed_job.#{worker_index}"
+            run_process(process_name, dir)
+          end
         end
+        threads.each(&:join)
       end
     end
 

data/lib/delayed/compatibility.rb

@@ -0,0 +1,27 @@
+require 'active_support/version'
+
+module Delayed
+  module Compatibility
+    if ActiveSupport::VERSION::MAJOR >= 4
+      require 'active_support/proxy_object'
+
+      def self.executable_prefix
+        'bin'
+      end
+
+      def self.proxy_object_class
+        ActiveSupport::ProxyObject
+      end
+    else
+      require 'active_support/basic_object'
+
+      def self.executable_prefix
+        'script'
+      end
+
+      def self.proxy_object_class
+        ActiveSupport::BasicObject
+      end
+    end
+  end
+end

data/lib/delayed/message_sending.rb

@@ -1,19 +1,7 @@
-if ActiveSupport::VERSION::MAJOR == 4
-  require 'active_support/proxy_object'
-else
-  require 'active_support/basic_object'
-end
-
 require 'active_support/core_ext/module/aliasing'
 
 module Delayed
-  if ActiveSupport::VERSION::MAJOR == 4
-    klass = ActiveSupport::ProxyObject
-  else
-    klass = ActiveSupport::BasicObject
-  end
-
-  class DelayProxy < klass
+  class DelayProxy < Delayed::Compatibility.proxy_object_class
     def initialize(payload_class, target, options)
       @payload_class = payload_class
       @target = target

data/lib/delayed/psych_ext.rb

@@ -1,11 +1,18 @@
 if defined?(ActiveRecord)
   ActiveRecord::Base.class_eval do
-    def encode_with_override(coder)
-      encode_with_without_override(coder)
-      coder.tag = "!ruby/ActiveRecord:#{self.class.name}"
+    if instance_methods.include?(:encode_with)
+      def encode_with_override(coder)
+        encode_with_without_override(coder)
+        coder.tag = "!ruby/ActiveRecord:#{self.class.name}"
+      end
+      alias_method :encode_with_without_override, :encode_with
+      alias_method :encode_with, :encode_with_override
+    else
+      def encode_with(coder)
+        coder["attributes"] = attributes
+        coder.tag = "!ruby/ActiveRecord:#{self.class.name}"
+      end
     end
-    alias_method :encode_with_without_override, :encode_with
-    alias_method :encode_with, :encode_with_override
   end
 end
 

data/lib/delayed/tasks.rb

@@ -22,4 +22,17 @@ namespace :jobs do
       :quiet => false
     }
   end
+
+  desc "Exit with error status if any jobs older than max_age seconds haven't been attempted yet."
+  task :check, [:max_age] => :environment do |_, args|
+    args.with_defaults(:max_age => 300)
+
+    unprocessed_jobs = Delayed::Job.where('attempts = 0 AND created_at < ?', Time.now - args[:max_age].to_i).count
+
+    if unprocessed_jobs > 0
+      fail "#{unprocessed_jobs} jobs older than #{args[:max_age]} seconds have not been processed yet"
+    end
+
+  end
+
 end

data/lib/delayed/worker.rb

@@ -8,6 +8,7 @@ require 'benchmark'
 
 module Delayed
   class Worker
+    DEFAULT_LOG_LEVEL        = Logger::INFO
     DEFAULT_SLEEP_DELAY      = 5
     DEFAULT_MAX_ATTEMPTS     = 25
     DEFAULT_MAX_RUN_TIME     = 4.hours
@@ -115,7 +116,7 @@ module Delayed
     end
 
     # Every worker has a unique name which by default is the pid of the process. There are some
-    # advantages to overriding this with something which survives worker retarts:  Workers can#
+    # advantages to overriding this with something which survives worker restarts:  Workers can
     # safely resume working on tasks which are locked by themselves. The worker will assume that
     # it crashed before.
     def name
@@ -162,7 +163,7 @@ module Delayed
               sleep(self.class.sleep_delay) unless stop?
             end
           else
-            say "#{count} jobs processed at %.4f j/s, %d failed ..." % [count / @realtime, @result.last]
+            say "#{count} jobs processed at %.4f j/s, %d failed" % [count / @realtime, @result.last]
           end
 
           break if stop?
@@ -199,11 +200,12 @@ module Delayed
     end
 
     def run(job)
+      job_say job, 'RUNNING'
       runtime =  Benchmark.realtime do
         Timeout.timeout(self.class.max_run_time.to_i, WorkerTimeout) { job.invoke_job }
         job.destroy
       end
-      say "#{job.name} completed after %.4f" % runtime
+      job_say job, 'COMPLETED after %.4f' % runtime
       return true  # did work
     rescue DeserializationError => error
       job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
@@ -222,7 +224,7 @@ module Delayed
         job.unlock
         job.save!
       else
-        say "PERMANENTLY removing #{job.name} because of #{job.attempts} consecutive failures.", Logger::INFO
+        job_say job, "REMOVED permanently because of #{job.attempts} consecutive failures", Logger::ERROR
         failed(job)
       end
     end
@@ -234,7 +236,12 @@ module Delayed
       end
     end
 
-    def say(text, level = Logger::INFO)
+    def job_say(job, text, level = DEFAULT_LOG_LEVEL)
+      text = "Job #{job.name} (id=#{job.id}) #{text}"
+      say text, level
+    end
+
+    def say(text, level = DEFAULT_LOG_LEVEL)
       text = "[Worker(#{name})] #{text}"
       puts text unless @quiet
       logger.add level, "#{Time.now.strftime('%FT%T%z')}: #{text}" if logger
@@ -248,7 +255,7 @@ module Delayed
 
     def handle_failed_job(job, error)
       job.last_error = "#{error.message}\n#{error.backtrace.join("\n")}"
-      say "#{job.name} failed with #{error.class.name}: #{error.message} - #{job.attempts} failed attempts", Logger::ERROR
+      job_say job, "FAILED (#{job.attempts} prior attempts) with #{error.class.name}: #{error.message}", Logger::ERROR
       reschedule(job)
     end
 
@@ -256,7 +263,7 @@ module Delayed
     # If no jobs are left we return nil
     def reserve_and_run_one_job
       job = Delayed::Job.reserve(self)
-      self.class.lifecycle.run_callbacks(:perform, self, job){ result = run(job) } if job
+      self.class.lifecycle.run_callbacks(:perform, self, job){ run(job) } if job
     end
   end
 

data/lib/delayed_job.rb

@@ -1,4 +1,5 @@
 require 'active_support'
+require 'delayed/compatibility'
 require 'delayed/exceptions'
 require 'delayed/message_sending'
 require 'delayed/performable_method'

data/lib/generators/delayed_job/delayed_job_generator.rb

@@ -1,11 +1,12 @@
 require 'rails/generators'
+require 'delayed/compatibility'
 
 class DelayedJobGenerator < Rails::Generators::Base
 
   self.source_paths << File.join(File.dirname(__FILE__), 'templates')
 
-  def create_script_file
-    template 'script', 'script/delayed_job'
-    chmod 'script/delayed_job', 0755
+  def create_executable_file
+    template "script", "#{Delayed::Compatibility.executable_prefix}/delayed_job"
+    chmod "#{Delayed::Compatibility.executable_prefix}/delayed_job", 0755
   end
 end

data/spec/helper.rb

@@ -1,12 +1,3 @@
-require 'simplecov'
-require 'coveralls'
-
-SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
-  SimpleCov::Formatter::HTMLFormatter,
-  Coveralls::SimpleCov::Formatter
-]
-SimpleCov.start
-
 require 'logger'
 require 'rspec'
 
@@ -17,6 +8,15 @@ require 'active_record'
 require 'delayed_job'
 require 'delayed/backend/shared_spec'
 
+require 'simplecov'
+require 'coveralls'
+
+SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+  SimpleCov::Formatter::HTMLFormatter,
+  Coveralls::SimpleCov::Formatter
+]
+SimpleCov.start
+
 Delayed::Worker.logger = Logger.new('/tmp/dj.log')
 ENV['RAILS_ENV'] = 'test'
 

data/spec/worker_spec.rb

@@ -21,6 +21,19 @@ describe Delayed::Worker do
     end
   end
 
+  describe "job_say" do
+    before do
+      @worker = Delayed::Worker.new
+      @job = stub('job', :id => 123, :name => 'ExampleJob')
+    end
+
+    it "logs with job name and id" do
+      @worker.should_receive(:say).
+        with('Job ExampleJob (id=123) message', Delayed::Worker::DEFAULT_LOG_LEVEL)
+      @worker.job_say(@job, 'message')
+    end
+  end
+
   context "worker read-ahead" do
     before do
       @read_ahead = Delayed::Worker.read_ahead

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: delayed_job
 version: !ruby/object:Gem::Version
-  version: 4.0.0.beta1
-  prerelease: 6
+  version: 4.0.0.beta2
 platform: ruby
 authors:
 - Brandon Keepers
@@ -16,12 +15,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-02 00:00:00.000000000 Z
+date: 2013-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  prerelease: false
   name: activesupport
-  requirement: !ruby/object:Gem::Requirement
-    none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -29,10 +28,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: '4.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -40,6 +36,7 @@ dependencies:
     - - <
       - !ruby/object:Gem::Version
         version: '4.1'
+  type: :runtime
 description: Delayed_job (or DJ) encapsulates the common pattern of asynchronously
   executing longer tasks in the background. It is a direct extraction from Shopify
   where the job table is responsible for a multitude of core tasks.
@@ -57,9 +54,12 @@ files:
 - delayed_job.gemspec
 - contrib/delayed_job.monitrc
 - contrib/delayed_job_multiple.monitrc
+- contrib/delayed_job_rails_4.monitrc
+- contrib/delayed_job_rails_4_multiple.monitrc
 - lib/delayed/backend/base.rb
 - lib/delayed/backend/shared_spec.rb
 - lib/delayed/command.rb
+- lib/delayed/compatibility.rb
 - lib/delayed/deserialization_error.rb
 - lib/delayed/exceptions.rb
 - lib/delayed/lifecycle.rb
@@ -98,30 +98,26 @@ files:
 homepage: http://github.com/collectiveidea/delayed_job
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -646849104774513769
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>'
     - !ruby/object:Gem::Version
       version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Database-backed asynchronous priority queue system -- Extracted from Shopify
 test_files:
 - spec/autoloaded/clazz.rb