delayed_job_active_record_threaded 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b4a6bd62a6a5e97dd63f75c7d18de220d1494304
+  data.tar.gz: 308d9aed19966440ae784559d7651b07675884a4
+SHA512:
+  metadata.gz: a2798cb2e8f68d171724c34d8114ec2e284c797191463804a17a3e1230a8c04c69c4727ed85dceaa68acf61b471984c8ba3f262008519d32b8f9bf1d56326eac
+  data.tar.gz: 51afdb426a5b47f0b0d6aa978ef3899a6b0a6a24133a98fefcc2f771a24e8e6a73b58185e2e3b19008a2d06043ac3359a10173a936565fe2d5e2c725db7afc0f

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.idea/workspace.xml
+.idea/

data/.travis.yml

@@ -0,0 +1,13 @@
+language: ruby
+only:
+  - master
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+env:
+  matrix:
+    - "RAILS_VERSION=\"~> 3.0.0\""
+    - "RAILS_VERSION=\"~> 3.1.0\""
+    - "RAILS_VERSION=\"~> 3.2.0\""
+    - "RAILS_VERSION=\"~> 4.0.0\""

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in delayed_job_active_record_threaded.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Abdo Achkar
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# DelayedJobActiveRecordThreaded
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'delayed_job_active_record_threaded'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install delayed_job_active_record_threaded
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,91 @@
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+require 'active_record'
+require 'active_support'
+require 'active_support/core_ext'
+require 'delayed_job'
+
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.pattern = "test/*_test.rb"
+end
+
+
+def prepare_connection
+  ENV["RAILS_ENV"] = "test"
+
+  #ENV["ADAPTER"] ||= "mysql2"
+
+  db_adapter, gemfile = ENV["ADAPTER"], ENV["BUNDLE_GEMFILE"]
+  db_adapter ||= gemfile && gemfile[%r(gemfiles/(.*?)/)] && $1
+  db_adapter ||= 'sqlite3'
+
+  config = YAML.load(File.read('test/database.yml'))
+  ActiveRecord::Base.establish_connection config[db_adapter]
+  ActiveRecord::Base.logger = $logger
+  ActiveRecord::Migration.verbose = false
+end
+
+task :prepare do
+  prepare_connection
+
+  Dir["db/migrate/**/*.rb"].each do |f|
+    File.delete "#{File.dirname(__FILE__)}/#{f}"
+  end
+
+  require "#{File.dirname(__FILE__)}/lib/generators/delayed_job/active_record_generator"
+  DelayedJob::ActiveRecordGenerator.new.create_migration_file
+
+  require "#{File.dirname(__FILE__)}/lib/delayed/job"
+
+
+  ::ActiveRecord::Base.clear_all_connections!
+end
+
+task :seed do
+  prepare_connection
+
+  require "#{File.dirname(__FILE__)}/lib/delayed/job"
+
+  def do_once
+    1000.times {
+      j = Delayed::Job.new
+      j.save
+    }
+  end
+
+  do_once
+
+  ::ActiveRecord::Base.clear_all_connections!
+end
+
+task :migrate do
+  prepare_connection
+
+  Dir["db/migrate/**/*.rb"].each do |f|
+    require "#{File.dirname(__FILE__)}/#{f}"
+
+    k = f.split("_")[1..-1].join("_").split(".")[0].camelize.constantize
+    puts "Migrating #{k.name}"
+    k.up
+    puts "Done."
+  end
+
+  ::ActiveRecord::Base.clear_all_connections!
+end
+
+task :rollback do
+  prepare_connection
+
+  Dir["db/migrate/**/*.rb"].each do |f|
+    require "#{File.dirname(__FILE__)}/#{f}"
+
+    k = f.split("_")[1..-1].join("_").split(".")[0].camelize.constantize
+    puts "Migrating #{k.name}"
+    k.down
+    puts "Done."
+  end
+
+  ::ActiveRecord::Base.clear_all_connections!
+end

data/delayed_job_active_record_threaded.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "delayed_job_active_record_threaded"
+  spec.authors       = ["Abdo Achkar"]
+  spec.email         = ["abdo.achkar@gmail.com"]
+  spec.description   = %q{Allows going through delayed job queues using threads instead of processes}
+  spec.summary       = %q{Process the delayed job queue using threads.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "minitest", "~> 4.7.3"
+  spec.add_development_dependency "rails", ['>= 3.0', '< 4.1']
+  spec.add_development_dependency "turn"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'sqlite3-ruby', '>= 1.3.1'
+  spec.add_dependency   'activerecord', ['>= 3.0', '< 4.1']
+  spec.add_dependency   'delayed_job',  ['>= 3.0', '< 4.1']
+  spec.add_dependency   'celluloid', ['>=0.14.1']
+  spec.version = "0.0.1"
+end

data/lib/delayed/fake_job.rb

@@ -0,0 +1,35 @@
+module Delayed
+  class FakeJob
+    @@count = 1
+    @@lock = Mutex.new
+    attr_accessor :run_at, :attempts, :object_id, :object_type, :id, :queue
+
+    def initialize(queue=nil)
+      @run_at = DateTime.now
+      @attempts = 0
+      @queue = queue
+
+      @@lock.synchronize {
+        @id = @@count
+        @@count += 1
+      }
+    end
+
+    def invoke_job
+      DelayedJobActiveRecordThreaded.logger.info "invoking job #{@id} in queue #{@queue}" if DelayedJobActiveRecordThreaded.logger
+      sleep(rand * 5)
+    end
+
+    def perform
+      invoke_job
+    end
+
+    def delete
+      DelayedJobActiveRecordThreaded.logger.info "deleting job #{@id}" if DelayedJobActiveRecordThreaded.logger
+    end
+
+    def save
+      DelayedJobActiveRecordThreaded.logger "saving job #{@id}" if DelayedJobActiveRecordThreaded.logger
+    end
+  end
+end

data/lib/delayed/home_manager.rb

@@ -0,0 +1,102 @@
+require 'date_core'
+require 'celluloid'
+require 'timers'
+
+module Delayed
+  class HomeManager
+    include Celluloid
+
+    DEFAULT_SLEEP_TIME = 2
+    DEFAULT_JOBS_TO_PULL = 20
+    DEFAULT_WORKERS_NUMBER = 16
+    DEFAULT_TIMEOUT = 20
+    DEFAULT_MAX_ATTEMPTS = 25
+
+    attr_accessor :alive, :timeout, :sleep_time, :workers_number, :queue, :workers_pool, :max_attempts, :worker_options, :timer, :hostname
+
+    def initialize(options=nil)
+      self.alive = true
+
+      if (options && options.class == Hash)
+        @sleep_time = options[:sleep_time]
+        @workers_number = options[:workers_number]
+        @worker_options = options[:worker_options]
+        @queue = options[:queue]
+        @timeout = options[:timeout]
+        @max_attempts = options[:max_attempts]
+      end
+
+      @sleep_time ||= DEFAULT_SLEEP_TIME
+      @timeout ||= DEFAULT_TIMEOUT
+      @workers_number ||= DEFAULT_WORKERS_NUMBER
+      @max_attempts ||= DEFAULT_MAX_ATTEMPTS
+      @worker_options ||= {}
+    end
+
+    def start
+      # use Celluloid to create a pool of size @workers_number
+      # worker_options get passed to HomeWorker's initializer
+      @workers_pool = HomeWorker.pool(:size => @workers_number, :args => @worker_options)
+
+      begin
+        # make sure jobs locked by our hostname in prior attempts are unlocked
+        unlock_all
+      rescue
+        DelayedJobActiveRecordThreaded.logger.error($!.message) if DelayedJobActiveRecordThreaded.logger
+        DelayedJobActiveRecordThreaded.logger.error($!.backtrace.join("\n")) if DelayedJobActiveRecordThreaded.logger
+      end
+
+      @timer = every(@sleep_time) {
+        begin
+          if (!@alive)
+            @timer.stop
+          end
+
+          jobs = pull_next(@queue, @workers_pool.idle_size)
+          jobs.each { |j| @workers_pool.async.work(j) }
+        rescue
+          # logging error watch
+          begin
+            DelayedJobActiveRecordThreaded.logger.error($!.message) if DelayedJobActiveRecordThreaded.logger
+            DelayedJobActiveRecordThreaded.logger.error($!.backtrace.join("\n")) if DelayedJobActiveRecordThreaded.logger
+          rescue
+          end
+        end
+      }
+    end
+
+    # Unlock all jobs locked by our hostname in prior attempts
+    def unlock_all
+      Delayed::Job.transaction do
+        Delayed::Job.where(:locked_by => hostname).update_all(:locked_by => nil, :locked_at => nil)
+      end
+    end
+
+    # pull n items from Delayed::Job
+    # locks jobs until they're processed (the worker then deletes the job)
+    def pull_next(queue=nil, n=15)
+      ids = []
+      Delayed::Job.transaction do
+        query = Delayed::Job.where("(run_at is null or run_at < ?) and locked_at is null", DateTime.now).order("priority asc, run_at asc, id asc")
+        if (queue)
+          query = query.where(:queue => queue)
+        end
+
+        query = query.limit(n)
+        ids = query.pluck(:id)
+        query.update_all(:locked_at => DateTime.now.utc, :locked_by => hostname)
+      end
+
+      return Delayed::Job.where(:id => ids)
+    end
+
+    def hostname
+      @hostname ||= Socket.gethostname
+      return @hostname
+    end
+
+    def kill
+      @alive = false
+    end
+  end
+end

data/lib/delayed/home_worker.rb

@@ -0,0 +1,62 @@
+require 'timeout'
+require 'celluloid'
+
+module Delayed
+  class HomeWorker
+    include Celluloid
+
+    DEFAULT_MAX_ATTEMPTS = 25
+    DEFAULT_TIMEOUT = 60
+
+    attr_accessor :timeout, :max_attempts
+
+    def initialize(options=nil)
+      if (options && options.class == Hash)
+        @max_attempts = options[:max_attempts]
+        @timeout = options[:timeout]
+      end
+
+      @max_attempts ||= DEFAULT_MAX_ATTEMPTS
+      @timeout ||= DEFAULT_TIMEOUT
+    end
+
+    def work(job)
+      delayable_type = job.delayable_type
+      delayable_id = job.delayable_id
+      jid = job.id
+
+      t = DateTime.now.to_f
+      if (job && job.respond_to?(:invoke_job))
+        begin
+          Timeout.timeout(@timeout) do
+            job.invoke_job
+            job.delete
+          end
+        rescue
+          if (job.attempts <= @max_attempts)
+            job.attempts += 1
+            job.run_at = (job.attempts ** 4 + 5).seconds.from_now
+            job.save
+          end
+        end
+
+        delta = DateTime.now.to_f - t
+
+        if delayable_type && delayable_id
+          DelayedJobActiveRecordThreaded.logger.info "[#{DateTime.now.to_s}] Job for #{delayable_type} delayable_id #{delayable_id} completed in #{delta} seconds" if DelayedJobActiveRecordThreaded.logger
+        else
+          DelayedJobActiveRecordThreaded.logger.info "[#{DateTime.now.to_s}] Job #{jid} completed in #{delta} seconds" if DelayedJobActiveRecordThreaded.logger
+        end
+      end
+
+    ensure
+      #attempt closing connection
+      begin
+        if (ActiveRecord::Base.connection && ActiveRecord::Base.connection.active?)
+          ActiveRecord::Base.connection.close
+        end
+      rescue
+      end
+    end
+  end
+end

data/lib/delayed/job.rb

@@ -0,0 +1,14 @@
+require 'active_record'
+require 'delayed_job'
+
+module Delayed
+  # A job object that is persisted to the database.
+  # Contains the work object as a YAML field.
+  class Job < ::ActiveRecord::Base
+    include Delayed::Backend::Base
+
+    attr_accessible :queue, :priority, :payload_object, :delayable_id, :delayable_type
+
+    self.table_name = "delayed_jobs"
+  end
+end

data/lib/delayed/railtie.rb

@@ -0,0 +1,10 @@
+require 'delayed_job_active_record_threaded'
+require 'rails'
+
+module Delayed
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "#{File.dirname(__FILE__)}/../tasks/dj.rake"
+    end
+  end
+end

data/lib/delayed_job_active_record_threaded.rb

@@ -0,0 +1,32 @@
+Dir["tasks/**/*.rake"].each { |ext| load ext } if defined?(Rake)
+
+class DelayedJobActiveRecordThreaded
+  class << self
+    attr_accessor :logger
+  end
+
+  # if rails logger exists, use it
+  # otherwise, use STDOUT and STDERR
+  if defined?(Rails) && Rails.logger
+    DelayedJobActiveRecordThreaded.logger = Rails.logger if Rails.logger
+  else
+    DelayedJobActiveRecordThreaded.logger = Object.new
+
+    def logger.info(str)
+      STDOUT.write("#{str}\n");
+      nil
+    end
+
+    def logger.error(str)
+      STDERR.write("#{str}\n");
+      nil
+    end
+  end
+end
+
+to_require = %w(home_manager home_worker fake_job job)
+to_require.each do |f|
+  require "#{File.dirname(__FILE__)}/delayed/#{f}"
+end
+
+require "#{File.dirname(__FILE__)}/delayed/railtie" if defined?(Rails)

data/lib/generators/delayed_job/active_record_generator.rb

@@ -0,0 +1,21 @@
+# copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/active_record_generator.rb
+
+require 'rails/generators/migration'
+require 'rails/generators/active_record'
+
+# Extend the DelayedJobGenerator so that it creates an AR migration
+module DelayedJob
+  class ActiveRecordGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+    def create_migration_file
+      migration_template 'migration.rb', 'db/migrate/create_delayed_jobs.rb'
+    end
+
+    def self.next_migration_number dirname
+      ActiveRecord::Generators::Base.next_migration_number dirname
+    end
+  end
+end

data/lib/generators/delayed_job/next_migration_version.rb

@@ -0,0 +1,16 @@
+# copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/next_migration_version.rb
+
+module DelayedJob
+  module NextMigrationVersion
+    # while methods have moved around this has been the implementation
+    # since ActiveRecord 3.0
+    def next_migration_number(dirname)
+      next_migration_number = current_migration_number(dirname) + 1
+      if ActiveRecord::Base.timestamped_migrations
+        [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
+      else
+        "%.3d" % next_migration_number
+      end
+    end
+  end
+end

data/lib/generators/delayed_job/templates/migration.rb

@@ -0,0 +1,27 @@
+# copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/templates/migration.rb
+
+class CreateDelayedJobs < ActiveRecord::Migration
+  def self.up
+    create_table :delayed_jobs, :force => true do |table|
+      table.integer  :priority, :default => 0      # Allows some jobs to jump to the front of the queue
+      table.integer  :attempts, :default => 0      # Provides for retries, but still fail eventually.
+      table.text     :handler                      # YAML-encoded string of the object that will do work
+      table.text     :last_error                   # reason for last failure (See Note below)
+      table.datetime :run_at                       # When to run. Could be Time.zone.now for immediately, or sometime in the future.
+      table.datetime :locked_at                    # Set when a client is working on this object
+      table.datetime :failed_at                    # Set when all retries have failed (actually, by default, the record is deleted instead)
+      table.string   :locked_by                    # Who is working on this object (if locked)
+      table.string   :queue                        # The name of the queue this job is in
+      table.integer  :delayable_id                 # Store the id of a delayable ActiveRecord object. This is helpful when you wish to check if a certain object exists in the queue
+      table.string   :delayable_type               # The class name of the delayable ActiveRecord object
+      table.timestamps
+    end
+
+    add_index :delayed_jobs, [:failed_at, :priority, :run_at], :name => 'delayed_jobs_priority'
+    add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_object'
+  end
+
+  def self.down
+    drop_table :delayed_jobs
+  end
+end

data/lib/generators/delayed_job/templates/upgrade_migration.rb

@@ -0,0 +1,18 @@
+# copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/templates/upgrade_migration.rb
+
+class AddColumnsToDelayedJobs < ActiveRecord::Migration
+  def self.up
+    add_column :delayed_jobs, :delayable_id, :integer
+    add_column :delayed_jobs, :delayable_type, :string
+    remove_index :delayed_jobs, :name => "delayed_jobs_priority"
+    add_index :delayed_jobs, [:failed_at, :priority, :run_at], :name => 'delayed_jobs_priority'
+    add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_object'
+  end
+
+  def self.down
+    remove_column :delayed_jobs, :delayable_id
+    remove_column :delayed_jobs, :delayable_type
+    remove_index :delayed_jobs, :name => "delayed_jobs_priority"
+    add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+  end
+end

data/lib/generators/delayed_job/upgrade_generator.rb

@@ -0,0 +1,19 @@
+# copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/upgrade_generator.rb
+
+#require 'generators/delayed_job/delayed_job_generator'
+require 'rails/generators/migration'
+require 'rails/generators/active_record/migration'
+
+# Extend the DelayedJobGenerator so that it creates an AR migration
+module DelayedJob
+  class UpgradeGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+    extend ActiveRecord::Generators::Migration
+
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+
+    def create_migration_file
+      migration_template 'upgrade_migration.rb', 'db/migrate/add_columns_to_delayed_jobs.rb'
+    end
+  end
+end

data/lib/tasks/dj.rake

@@ -0,0 +1,103 @@
+require 'rack/utils'
+
+namespace :dj do
+  task :run, [:args_expr ] => :environment do |t,args|
+    Rake::Task["delayed_job:run"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
+  end
+
+  task :start, [:args_expr ] => :environment do |t,args|
+    Rake::Task["delayed_job:start"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
+  end
+
+  task :stop, [:args_expr ] => :environment do |t,args|
+    Rake::Task["delayed_job:stop"].invoke
+  end
+
+  task :kill_all, [:args_expr ] => :environment do |t,args|
+    Rake::Task["delayed_job:kill_all"].invoke
+  end
+
+  task :print_options, [:args_expr ] => :environment do |t,args|
+    Rake::Task["delayed_job:print_options"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
+  end
+end
+
+namespace :delayed_job do
+  # allow passing args like querystring
+  # parse args with Rack::Utils.parse_nested_query(query)
+  # (must require 'rack/utils')
+  # rake "delayed_job:run[default[workers_number]=16&default[worker_timeout]=120]"
+  task :run, [ :args_expr ] => :environment do |t, args|
+    DEFAULT_QUEUE_NAME = nil
+    DEFAULT_WORKERS_NUMBER = 16
+    DEFAULT_WORKERS_TIMEOUT = 60
+
+    args.with_defaults(:args_expr => "default[workers_number]=16&default[worker_timeout]=60")
+    puts "args: #{args[:args_expr]}"
+
+    options = Rack::Utils.parse_nested_query(args[:args_expr])
+    puts "options #{options}"
+
+    options.keys.each do |k|
+      queue_name = k
+      if options[k]
+        workers_number = options[k]["workers_number"]
+        workers_timeout = options[k]["worker_timeout"]
+        # in case param is misspelled
+        workers_timeout ||= options[k]["workers_timeout"]
+      end
+
+      queue_name ||= DEFAULT_QUEUE_NAME
+      workers_number ||= DEFAULT_WORKERS_NUMBER
+      workers_timeout ||= DEFAULT_WORKERS_TIMEOUT
+
+      # set to nil of queue_name is default
+      queue_name = nil if (queue_name == "default")
+
+      puts "queue: #{queue_name}, workers_number: #{workers_number}, workers_timeout: #{workers_timeout}"
+
+      Delayed::HomeManager.new({ :workers_number => workers_number.to_i, :queue => queue_name, :worker_options => { :timeout => workers_timeout.to_i } }).start
+    end
+
+    # stay alive
+    while(true)
+      sleep(5)
+    end
+  end
+
+  def dj_pid
+    return 'tmp/pids/delayed_job.pid'
+  end
+
+  task :start, [ :args_expr ] => :environment do |t, args|
+    puts "#{args[:args_expr]}"
+
+    cmd = %(if [ -f #{dj_pid} ] && [ -n `cat #{dj_pid}` ] && [ ps -p `cat #{dj_pid}` > /dev/null ]; then sudo kill -9 `cat #{dj_pid}`; fi
+            (bundle exec rake "delayed_job:run[#{args[:args_expr]}]" >> log/delayed_job.log 2>&1) & (echo $! > tmp/pids/dj.pid)
+           )
+
+    puts "executing: #{cmd}"
+
+    # execute cmd
+    %x(#{cmd})
+  end
+
+  task :stop => :environment do |t, args|
+    cmd = %(if [ -f #{dj_pid} ] && [ -n `cat #{dj_pid}` ] && [ ps -p `cat #{dj_pid}` > /dev/null ]; then kill -9 `cat #{dj_pid}`; fi)
+
+    # execute cmd
+    %x(#{cmd})
+  end
+
+  task :print_options, [ :args_expr ] => :environment do |t, args|
+    args.with_defaults(:args_expr => "name=abdo&fruits[]=bananas&fruits[]=dates")
+    options = Rack::Utils.parse_nested_query(args[:args_expr])
+
+    puts options
+  end
+
+  task :kill_all do
+    cmd = %(ps aux | grep delayed_job | awk -F" " '{ print $2 }' | xargs kill -9)
+    %x(#{cmd})
+  end
+end

data/test/database.yml

@@ -0,0 +1,15 @@
+mysql2:
+  adapter: mysql2
+  database: delayed_job_test
+  username: dj
+  encoding: utf8
+  pool: 300
+
+postgresql:
+  adapter: postgresql
+  database: delayed_job_test
+  username: postgres
+
+sqlite3:
+  adapter: sqlite3
+  database: ":memory:"

data/test/delayed_job_active_record_threaded_test.rb

@@ -0,0 +1,47 @@
+require "test_helper"
+require "delayed_job_active_record_threaded"
+require "rails"
+require "rails/test_help"
+require "minitest/rails"
+
+class TestJob < Struct.new(:id)
+  def perform
+    story = Story.find(id)
+    sleep(rand(2))
+    puts "Telling #{story.text}\n" if story
+  end
+end
+
+class DelayedJobActiveRecordThreadedTest < ActiveSupport::TestCase #MiniTest::Unit::TestCase
+  self.use_transactional_fixtures = false
+
+  QUEUE_NAME = "StoriesQueue"
+
+  before do
+    #Delayed::Job.new(:run_at => 10.seconds.ago).save
+    100.times { |i|
+      s = Story.create!(:text => "Story #{i}")
+      Delayed::Job.enqueue(TestJob.new(s.id), :queue => QUEUE_NAME, :priority => i, :delayable_id => s.id, :delayable_type => s.class.name, :run_at => 10.seconds.ago)
+    }
+  end
+
+  after do
+  end
+
+  # this is VERY dependent on the database pool size
+  test "should process jobs without crashing" do
+    mgr = Delayed::HomeManager.new({ :sleep_time => 1, :workers_number => 25, :queue => QUEUE_NAME })
+
+    puts "Number of jobs in queue is: #{Delayed::Job.count}"
+
+    mgr.start
+
+    sleep(10)
+
+    mgr.kill
+
+    puts "Number of threads: #{Thread.list.count}"
+
+    assert Delayed::Job.where("queue = ?", QUEUE_NAME).count == 0, "Expected to have an empty queue after 10 seconds. Queue size was #{Delayed::Job.count} instead"
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,71 @@
+require 'minitest/unit'
+require 'minitest/autorun'
+require 'turn'
+
+
+require 'yaml'
+require 'active_record'
+require 'delayed_job'
+require "#{File.dirname(__FILE__)}/../lib/delayed_job_active_record_threaded"
+require "#{File.dirname(__FILE__)}/../lib/delayed/job"
+
+# copied (and modified) from https://github.com/collectiveidea/delayed_job_active_record/blob/master/spec/helper.rb
+begin
+  require 'protected_attributes'
+rescue LoadError
+end
+
+$logger = Logger.new('/tmp/dj.log')
+ENV['RAILS_ENV'] = 'test'
+
+db_adapter, gemfile = ENV["ADAPTER"], ENV["BUNDLE_GEMFILE"]
+db_adapter ||= gemfile && gemfile[%r(gemfiles/(.*?)/)] && $1
+#db_adapter ||= 'sqlite3'
+db_adapter ||= 'mysql2'
+
+config = YAML.load(File.read('test/database.yml'))
+ActiveRecord::Base.establish_connection config[db_adapter]
+ActiveRecord::Base.logger = $logger
+ActiveRecord::Migration.verbose = true
+
+ActiveRecord::Schema.define do
+  create_table :delayed_jobs, :force => true do |table|
+    table.integer  :priority, :default => 0
+    table.integer  :attempts, :default => 0
+    table.text     :handler
+    table.text     :last_error
+    table.datetime :run_at
+    table.datetime :locked_at
+    table.datetime :failed_at
+    table.string   :locked_by
+    table.string   :queue
+    table.integer  :delayable_id
+    table.string   :delayable_type
+    table.timestamps
+  end
+
+  add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
+  add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_delayable'
+
+  create_table :stories, :primary_key => :story_id, :force => true do |table|
+    table.string :text
+    table.boolean :scoped, :default => true
+  end
+end
+
+# Purely useful for test cases...
+class Story < ActiveRecord::Base
+  if ::ActiveRecord::VERSION::MAJOR < 4 && ActiveRecord::VERSION::MINOR < 2
+    set_primary_key :story_id
+  else
+    self.primary_key = :story_id
+  end
+  def tell; text; end
+  def whatever(n, _); tell*n; end
+  default_scope { where(:scoped => true) }
+
+  handle_asynchronously :whatever
+end
+
+# Add this directory so the ActiveSupport autoloading works
+ActiveSupport::Dependencies.autoload_paths << File.dirname(__FILE__)

metadata

@@ -0,0 +1,213 @@
+--- !ruby/object:Gem::Specification
+name: delayed_job_active_record_threaded
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Abdo Achkar
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 4.7.3
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.1
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: delayed_job
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - <
+      - !ruby/object:Gem::Version
+        version: '4.1'
+- !ruby/object:Gem::Dependency
+  name: celluloid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.14.1
+description: Allows going through delayed job queues using threads instead of processes
+email:
+- abdo.achkar@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- delayed_job_active_record_threaded.gemspec
+- lib/delayed/fake_job.rb
+- lib/delayed/home_manager.rb
+- lib/delayed/home_worker.rb
+- lib/delayed/job.rb
+- lib/delayed/railtie.rb
+- lib/delayed_job_active_record_threaded.rb
+- lib/generators/delayed_job/active_record_generator.rb
+- lib/generators/delayed_job/next_migration_version.rb
+- lib/generators/delayed_job/templates/migration.rb
+- lib/generators/delayed_job/templates/upgrade_migration.rb
+- lib/generators/delayed_job/upgrade_generator.rb
+- lib/tasks/dj.rake
+- test/database.yml
+- test/delayed_job_active_record_threaded_test.rb
+- test/test_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Process the delayed job queue using threads.
+test_files:
+- test/database.yml
+- test/delayed_job_active_record_threaded_test.rb
+- test/test_helper.rb