delayed_job_active_record_threaded 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: b4a6bd62a6a5e97dd63f75c7d18de220d1494304
4
+ data.tar.gz: 308d9aed19966440ae784559d7651b07675884a4
5
+ SHA512:
6
+ metadata.gz: a2798cb2e8f68d171724c34d8114ec2e284c797191463804a17a3e1230a8c04c69c4727ed85dceaa68acf61b471984c8ba3f262008519d32b8f9bf1d56326eac
7
+ data.tar.gz: 51afdb426a5b47f0b0d6aa978ef3899a6b0a6a24133a98fefcc2f771a24e8e6a73b58185e2e3b19008a2d06043ac3359a10173a936565fe2d5e2c725db7afc0f
data/.gitignore ADDED
@@ -0,0 +1,19 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ .idea/workspace.xml
19
+ .idea/
data/.travis.yml ADDED
@@ -0,0 +1,13 @@
1
+ language: ruby
2
+ only:
3
+ - master
4
+ rvm:
5
+ - 1.9.2
6
+ - 1.9.3
7
+ - 2.0.0
8
+ env:
9
+ matrix:
10
+ - "RAILS_VERSION=\"~> 3.0.0\""
11
+ - "RAILS_VERSION=\"~> 3.1.0\""
12
+ - "RAILS_VERSION=\"~> 3.2.0\""
13
+ - "RAILS_VERSION=\"~> 4.0.0\""
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in delayed_job_active_record_threaded.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2013 Abdo Achkar
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,29 @@
1
+ # DelayedJobActiveRecordThreaded
2
+
3
+ TODO: Write a gem description
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ gem 'delayed_job_active_record_threaded'
10
+
11
+ And then execute:
12
+
13
+ $ bundle
14
+
15
+ Or install it yourself as:
16
+
17
+ $ gem install delayed_job_active_record_threaded
18
+
19
+ ## Usage
20
+
21
+ TODO: Write usage instructions here
22
+
23
+ ## Contributing
24
+
25
+ 1. Fork it
26
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
27
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
28
+ 4. Push to the branch (`git push origin my-new-feature`)
29
+ 5. Create new Pull Request
data/Rakefile ADDED
@@ -0,0 +1,91 @@
1
+ require "bundler/gem_tasks"
2
+
3
+ require 'rake/testtask'
4
+ require 'active_record'
5
+ require 'active_support'
6
+ require 'active_support/core_ext'
7
+ require 'delayed_job'
8
+
9
+ Rake::TestTask.new do |t|
10
+ t.libs << 'test'
11
+ t.pattern = "test/*_test.rb"
12
+ end
13
+
14
+
15
+ def prepare_connection
16
+ ENV["RAILS_ENV"] = "test"
17
+
18
+ #ENV["ADAPTER"] ||= "mysql2"
19
+
20
+ db_adapter, gemfile = ENV["ADAPTER"], ENV["BUNDLE_GEMFILE"]
21
+ db_adapter ||= gemfile && gemfile[%r(gemfiles/(.*?)/)] && $1
22
+ db_adapter ||= 'sqlite3'
23
+
24
+ config = YAML.load(File.read('test/database.yml'))
25
+ ActiveRecord::Base.establish_connection config[db_adapter]
26
+ ActiveRecord::Base.logger = $logger
27
+ ActiveRecord::Migration.verbose = false
28
+ end
29
+
30
+ task :prepare do
31
+ prepare_connection
32
+
33
+ Dir["db/migrate/**/*.rb"].each do |f|
34
+ File.delete "#{File.dirname(__FILE__)}/#{f}"
35
+ end
36
+
37
+ require "#{File.dirname(__FILE__)}/lib/generators/delayed_job/active_record_generator"
38
+ DelayedJob::ActiveRecordGenerator.new.create_migration_file
39
+
40
+ require "#{File.dirname(__FILE__)}/lib/delayed/job"
41
+
42
+
43
+ ::ActiveRecord::Base.clear_all_connections!
44
+ end
45
+
46
+ task :seed do
47
+ prepare_connection
48
+
49
+ require "#{File.dirname(__FILE__)}/lib/delayed/job"
50
+
51
+ def do_once
52
+ 1000.times {
53
+ j = Delayed::Job.new
54
+ j.save
55
+ }
56
+ end
57
+
58
+ do_once
59
+
60
+ ::ActiveRecord::Base.clear_all_connections!
61
+ end
62
+
63
+ task :migrate do
64
+ prepare_connection
65
+
66
+ Dir["db/migrate/**/*.rb"].each do |f|
67
+ require "#{File.dirname(__FILE__)}/#{f}"
68
+
69
+ k = f.split("_")[1..-1].join("_").split(".")[0].camelize.constantize
70
+ puts "Migrating #{k.name}"
71
+ k.up
72
+ puts "Done."
73
+ end
74
+
75
+ ::ActiveRecord::Base.clear_all_connections!
76
+ end
77
+
78
+ task :rollback do
79
+ prepare_connection
80
+
81
+ Dir["db/migrate/**/*.rb"].each do |f|
82
+ require "#{File.dirname(__FILE__)}/#{f}"
83
+
84
+ k = f.split("_")[1..-1].join("_").split(".")[0].camelize.constantize
85
+ puts "Migrating #{k.name}"
86
+ k.down
87
+ puts "Done."
88
+ end
89
+
90
+ ::ActiveRecord::Base.clear_all_connections!
91
+ end
@@ -0,0 +1,29 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "delayed_job_active_record_threaded"
7
+ spec.authors = ["Abdo Achkar"]
8
+ spec.email = ["abdo.achkar@gmail.com"]
9
+ spec.description = %q{Allows going through delayed job queues using threads instead of processes}
10
+ spec.summary = %q{Process the delayed job queue using threads.}
11
+ spec.homepage = ""
12
+ spec.license = "MIT"
13
+
14
+ spec.files = `git ls-files`.split($/)
15
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
16
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
17
+ spec.require_paths = ["lib"]
18
+
19
+ spec.add_development_dependency "bundler", "~> 1.3"
20
+ spec.add_development_dependency "minitest", "~> 4.7.3"
21
+ spec.add_development_dependency "rails", ['>= 3.0', '< 4.1']
22
+ spec.add_development_dependency "turn"
23
+ spec.add_development_dependency "rake"
24
+ spec.add_development_dependency 'sqlite3-ruby', '>= 1.3.1'
25
+ spec.add_dependency 'activerecord', ['>= 3.0', '< 4.1']
26
+ spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.1']
27
+ spec.add_dependency 'celluloid', ['>=0.14.1']
28
+ spec.version = "0.0.1"
29
+ end
@@ -0,0 +1,35 @@
1
+ module Delayed
2
+ class FakeJob
3
+ @@count = 1
4
+ @@lock = Mutex.new
5
+ attr_accessor :run_at, :attempts, :object_id, :object_type, :id, :queue
6
+
7
+ def initialize(queue=nil)
8
+ @run_at = DateTime.now
9
+ @attempts = 0
10
+ @queue = queue
11
+
12
+ @@lock.synchronize {
13
+ @id = @@count
14
+ @@count += 1
15
+ }
16
+ end
17
+
18
+ def invoke_job
19
+ DelayedJobActiveRecordThreaded.logger.info "invoking job #{@id} in queue #{@queue}" if DelayedJobActiveRecordThreaded.logger
20
+ sleep(rand * 5)
21
+ end
22
+
23
+ def perform
24
+ invoke_job
25
+ end
26
+
27
+ def delete
28
+ DelayedJobActiveRecordThreaded.logger.info "deleting job #{@id}" if DelayedJobActiveRecordThreaded.logger
29
+ end
30
+
31
+ def save
32
+ DelayedJobActiveRecordThreaded.logger "saving job #{@id}" if DelayedJobActiveRecordThreaded.logger
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,102 @@
1
+ require 'date_core'
2
+ require 'celluloid'
3
+ require 'timers'
4
+
5
+ module Delayed
6
+ class HomeManager
7
+ include Celluloid
8
+
9
+ DEFAULT_SLEEP_TIME = 2
10
+ DEFAULT_JOBS_TO_PULL = 20
11
+ DEFAULT_WORKERS_NUMBER = 16
12
+ DEFAULT_TIMEOUT = 20
13
+ DEFAULT_MAX_ATTEMPTS = 25
14
+
15
+ attr_accessor :alive, :timeout, :sleep_time, :workers_number, :queue, :workers_pool, :max_attempts, :worker_options, :timer, :hostname
16
+
17
+ def initialize(options=nil)
18
+ self.alive = true
19
+
20
+ if (options && options.class == Hash)
21
+ @sleep_time = options[:sleep_time]
22
+ @workers_number = options[:workers_number]
23
+ @worker_options = options[:worker_options]
24
+ @queue = options[:queue]
25
+ @timeout = options[:timeout]
26
+ @max_attempts = options[:max_attempts]
27
+ end
28
+
29
+ @sleep_time ||= DEFAULT_SLEEP_TIME
30
+ @timeout ||= DEFAULT_TIMEOUT
31
+ @workers_number ||= DEFAULT_WORKERS_NUMBER
32
+ @max_attempts ||= DEFAULT_MAX_ATTEMPTS
33
+ @worker_options ||= {}
34
+ end
35
+
36
+ def start
37
+ # use Celluloid to create a pool of size @workers_number
38
+ # worker_options get passed to HomeWorker's initializer
39
+ @workers_pool = HomeWorker.pool(:size => @workers_number, :args => @worker_options)
40
+
41
+ begin
42
+ # make sure jobs locked by our hostname in prior attempts are unlocked
43
+ unlock_all
44
+ rescue
45
+ DelayedJobActiveRecordThreaded.logger.error($!.message) if DelayedJobActiveRecordThreaded.logger
46
+ DelayedJobActiveRecordThreaded.logger.error($!.backtrace.join("\n")) if DelayedJobActiveRecordThreaded.logger
47
+ end
48
+
49
+ @timer = every(@sleep_time) {
50
+ begin
51
+ if (!@alive)
52
+ @timer.stop
53
+ end
54
+
55
+ jobs = pull_next(@queue, @workers_pool.idle_size)
56
+ jobs.each { |j| @workers_pool.async.work(j) }
57
+ rescue
58
+ # logging error watch
59
+ begin
60
+ DelayedJobActiveRecordThreaded.logger.error($!.message) if DelayedJobActiveRecordThreaded.logger
61
+ DelayedJobActiveRecordThreaded.logger.error($!.backtrace.join("\n")) if DelayedJobActiveRecordThreaded.logger
62
+ rescue
63
+ end
64
+ end
65
+ }
66
+ end
67
+
68
+ # Unlock all jobs locked by our hostname in prior attempts
69
+ def unlock_all
70
+ Delayed::Job.transaction do
71
+ Delayed::Job.where(:locked_by => hostname).update_all(:locked_by => nil, :locked_at => nil)
72
+ end
73
+ end
74
+
75
+ # pull n items from Delayed::Job
76
+ # locks jobs until they're processed (the worker then deletes the job)
77
+ def pull_next(queue=nil, n=15)
78
+ ids = []
79
+ Delayed::Job.transaction do
80
+ query = Delayed::Job.where("(run_at is null or run_at < ?) and locked_at is null", DateTime.now).order("priority asc, run_at asc, id asc")
81
+ if (queue)
82
+ query = query.where(:queue => queue)
83
+ end
84
+
85
+ query = query.limit(n)
86
+ ids = query.pluck(:id)
87
+ query.update_all(:locked_at => DateTime.now.utc, :locked_by => hostname)
88
+ end
89
+
90
+ return Delayed::Job.where(:id => ids)
91
+ end
92
+
93
+ def hostname
94
+ @hostname ||= Socket.gethostname
95
+ return @hostname
96
+ end
97
+
98
+ def kill
99
+ @alive = false
100
+ end
101
+ end
102
+ end
@@ -0,0 +1,62 @@
1
+ require 'timeout'
2
+ require 'celluloid'
3
+
4
+ module Delayed
5
+ class HomeWorker
6
+ include Celluloid
7
+
8
+ DEFAULT_MAX_ATTEMPTS = 25
9
+ DEFAULT_TIMEOUT = 60
10
+
11
+ attr_accessor :timeout, :max_attempts
12
+
13
+ def initialize(options=nil)
14
+ if (options && options.class == Hash)
15
+ @max_attempts = options[:max_attempts]
16
+ @timeout = options[:timeout]
17
+ end
18
+
19
+ @max_attempts ||= DEFAULT_MAX_ATTEMPTS
20
+ @timeout ||= DEFAULT_TIMEOUT
21
+ end
22
+
23
+ def work(job)
24
+ delayable_type = job.delayable_type
25
+ delayable_id = job.delayable_id
26
+ jid = job.id
27
+
28
+ t = DateTime.now.to_f
29
+ if (job && job.respond_to?(:invoke_job))
30
+ begin
31
+ Timeout.timeout(@timeout) do
32
+ job.invoke_job
33
+ job.delete
34
+ end
35
+ rescue
36
+ if (job.attempts <= @max_attempts)
37
+ job.attempts += 1
38
+ job.run_at = (job.attempts ** 4 + 5).seconds.from_now
39
+ job.save
40
+ end
41
+ end
42
+
43
+ delta = DateTime.now.to_f - t
44
+
45
+ if delayable_type && delayable_id
46
+ DelayedJobActiveRecordThreaded.logger.info "[#{DateTime.now.to_s}] Job for #{delayable_type} delayable_id #{delayable_id} completed in #{delta} seconds" if DelayedJobActiveRecordThreaded.logger
47
+ else
48
+ DelayedJobActiveRecordThreaded.logger.info "[#{DateTime.now.to_s}] Job #{jid} completed in #{delta} seconds" if DelayedJobActiveRecordThreaded.logger
49
+ end
50
+ end
51
+
52
+ ensure
53
+ #attempt closing connection
54
+ begin
55
+ if (ActiveRecord::Base.connection && ActiveRecord::Base.connection.active?)
56
+ ActiveRecord::Base.connection.close
57
+ end
58
+ rescue
59
+ end
60
+ end
61
+ end
62
+ end
@@ -0,0 +1,14 @@
1
+ require 'active_record'
2
+ require 'delayed_job'
3
+
4
+ module Delayed
5
+ # A job object that is persisted to the database.
6
+ # Contains the work object as a YAML field.
7
+ class Job < ::ActiveRecord::Base
8
+ include Delayed::Backend::Base
9
+
10
+ attr_accessible :queue, :priority, :payload_object, :delayable_id, :delayable_type
11
+
12
+ self.table_name = "delayed_jobs"
13
+ end
14
+ end
@@ -0,0 +1,10 @@
1
+ require 'delayed_job_active_record_threaded'
2
+ require 'rails'
3
+
4
+ module Delayed
5
+ class Railtie < Rails::Railtie
6
+ rake_tasks do
7
+ load "#{File.dirname(__FILE__)}/../tasks/dj.rake"
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,32 @@
1
+ Dir["tasks/**/*.rake"].each { |ext| load ext } if defined?(Rake)
2
+
3
+ class DelayedJobActiveRecordThreaded
4
+ class << self
5
+ attr_accessor :logger
6
+ end
7
+
8
+ # if rails logger exists, use it
9
+ # otherwise, use STDOUT and STDERR
10
+ if defined?(Rails) && Rails.logger
11
+ DelayedJobActiveRecordThreaded.logger = Rails.logger if Rails.logger
12
+ else
13
+ DelayedJobActiveRecordThreaded.logger = Object.new
14
+
15
+ def logger.info(str)
16
+ STDOUT.write("#{str}\n");
17
+ nil
18
+ end
19
+
20
+ def logger.error(str)
21
+ STDERR.write("#{str}\n");
22
+ nil
23
+ end
24
+ end
25
+ end
26
+
27
+ to_require = %w(home_manager home_worker fake_job job)
28
+ to_require.each do |f|
29
+ require "#{File.dirname(__FILE__)}/delayed/#{f}"
30
+ end
31
+
32
+ require "#{File.dirname(__FILE__)}/delayed/railtie" if defined?(Rails)
@@ -0,0 +1,21 @@
1
+ # copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/active_record_generator.rb
2
+
3
+ require 'rails/generators/migration'
4
+ require 'rails/generators/active_record'
5
+
6
+ # Extend the DelayedJobGenerator so that it creates an AR migration
7
+ module DelayedJob
8
+ class ActiveRecordGenerator < Rails::Generators::Base
9
+ include Rails::Generators::Migration
10
+
11
+ self.source_paths << File.join(File.dirname(__FILE__), 'templates')
12
+
13
+ def create_migration_file
14
+ migration_template 'migration.rb', 'db/migrate/create_delayed_jobs.rb'
15
+ end
16
+
17
+ def self.next_migration_number dirname
18
+ ActiveRecord::Generators::Base.next_migration_number dirname
19
+ end
20
+ end
21
+ end
@@ -0,0 +1,16 @@
1
+ # copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/next_migration_version.rb
2
+
3
+ module DelayedJob
4
+ module NextMigrationVersion
5
+ # while methods have moved around this has been the implementation
6
+ # since ActiveRecord 3.0
7
+ def next_migration_number(dirname)
8
+ next_migration_number = current_migration_number(dirname) + 1
9
+ if ActiveRecord::Base.timestamped_migrations
10
+ [Time.now.utc.strftime("%Y%m%d%H%M%S"), "%.14d" % next_migration_number].max
11
+ else
12
+ "%.3d" % next_migration_number
13
+ end
14
+ end
15
+ end
16
+ end
@@ -0,0 +1,27 @@
1
+ # copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/templates/migration.rb
2
+
3
+ class CreateDelayedJobs < ActiveRecord::Migration
4
+ def self.up
5
+ create_table :delayed_jobs, :force => true do |table|
6
+ table.integer :priority, :default => 0 # Allows some jobs to jump to the front of the queue
7
+ table.integer :attempts, :default => 0 # Provides for retries, but still fail eventually.
8
+ table.text :handler # YAML-encoded string of the object that will do work
9
+ table.text :last_error # reason for last failure (See Note below)
10
+ table.datetime :run_at # When to run. Could be Time.zone.now for immediately, or sometime in the future.
11
+ table.datetime :locked_at # Set when a client is working on this object
12
+ table.datetime :failed_at # Set when all retries have failed (actually, by default, the record is deleted instead)
13
+ table.string :locked_by # Who is working on this object (if locked)
14
+ table.string :queue # The name of the queue this job is in
15
+ table.integer :delayable_id # Store the id of a delayable ActiveRecord object. This is helpful when you wish to check if a certain object exists in the queue
16
+ table.string :delayable_type # The class name of the delayable ActiveRecord object
17
+ table.timestamps
18
+ end
19
+
20
+ add_index :delayed_jobs, [:failed_at, :priority, :run_at], :name => 'delayed_jobs_priority'
21
+ add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_object'
22
+ end
23
+
24
+ def self.down
25
+ drop_table :delayed_jobs
26
+ end
27
+ end
@@ -0,0 +1,18 @@
1
+ # copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/templates/upgrade_migration.rb
2
+
3
+ class AddColumnsToDelayedJobs < ActiveRecord::Migration
4
+ def self.up
5
+ add_column :delayed_jobs, :delayable_id, :integer
6
+ add_column :delayed_jobs, :delayable_type, :string
7
+ remove_index :delayed_jobs, :name => "delayed_jobs_priority"
8
+ add_index :delayed_jobs, [:failed_at, :priority, :run_at], :name => 'delayed_jobs_priority'
9
+ add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_object'
10
+ end
11
+
12
+ def self.down
13
+ remove_column :delayed_jobs, :delayable_id
14
+ remove_column :delayed_jobs, :delayable_type
15
+ remove_index :delayed_jobs, :name => "delayed_jobs_priority"
16
+ add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
17
+ end
18
+ end
@@ -0,0 +1,19 @@
1
+ # copied from https://github.com/collectiveidea/delayed_job_active_record/blob/master/lib/generators/delayed_job/upgrade_generator.rb
2
+
3
+ #require 'generators/delayed_job/delayed_job_generator'
4
+ require 'rails/generators/migration'
5
+ require 'rails/generators/active_record/migration'
6
+
7
+ # Extend the DelayedJobGenerator so that it creates an AR migration
8
+ module DelayedJob
9
+ class UpgradeGenerator < Rails::Generators::Base
10
+ include Rails::Generators::Migration
11
+ extend ActiveRecord::Generators::Migration
12
+
13
+ self.source_paths << File.join(File.dirname(__FILE__), 'templates')
14
+
15
+ def create_migration_file
16
+ migration_template 'upgrade_migration.rb', 'db/migrate/add_columns_to_delayed_jobs.rb'
17
+ end
18
+ end
19
+ end
data/lib/tasks/dj.rake ADDED
@@ -0,0 +1,103 @@
1
+ require 'rack/utils'
2
+
3
+ namespace :dj do
4
+ task :run, [:args_expr ] => :environment do |t,args|
5
+ Rake::Task["delayed_job:run"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
6
+ end
7
+
8
+ task :start, [:args_expr ] => :environment do |t,args|
9
+ Rake::Task["delayed_job:start"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
10
+ end
11
+
12
+ task :stop, [:args_expr ] => :environment do |t,args|
13
+ Rake::Task["delayed_job:stop"].invoke
14
+ end
15
+
16
+ task :kill_all, [:args_expr ] => :environment do |t,args|
17
+ Rake::Task["delayed_job:kill_all"].invoke
18
+ end
19
+
20
+ task :print_options, [:args_expr ] => :environment do |t,args|
21
+ Rake::Task["delayed_job:print_options"].invoke(args[:args_expr].nil? ? args : args[:args_expr])
22
+ end
23
+ end
24
+
25
+ namespace :delayed_job do
26
+ # allow passing args like querystring
27
+ # parse args with Rack::Utils.parse_nested_query(query)
28
+ # (must require 'rack/utils')
29
+ # rake "delayed_job:run[default[workers_number]=16&default[worker_timeout]=120]"
30
+ task :run, [ :args_expr ] => :environment do |t, args|
31
+ DEFAULT_QUEUE_NAME = nil
32
+ DEFAULT_WORKERS_NUMBER = 16
33
+ DEFAULT_WORKERS_TIMEOUT = 60
34
+
35
+ args.with_defaults(:args_expr => "default[workers_number]=16&default[worker_timeout]=60")
36
+ puts "args: #{args[:args_expr]}"
37
+
38
+ options = Rack::Utils.parse_nested_query(args[:args_expr])
39
+ puts "options #{options}"
40
+
41
+ options.keys.each do |k|
42
+ queue_name = k
43
+ if options[k]
44
+ workers_number = options[k]["workers_number"]
45
+ workers_timeout = options[k]["worker_timeout"]
46
+ # in case param is misspelled
47
+ workers_timeout ||= options[k]["workers_timeout"]
48
+ end
49
+
50
+ queue_name ||= DEFAULT_QUEUE_NAME
51
+ workers_number ||= DEFAULT_WORKERS_NUMBER
52
+ workers_timeout ||= DEFAULT_WORKERS_TIMEOUT
53
+
54
+ # set to nil of queue_name is default
55
+ queue_name = nil if (queue_name == "default")
56
+
57
+ puts "queue: #{queue_name}, workers_number: #{workers_number}, workers_timeout: #{workers_timeout}"
58
+
59
+ Delayed::HomeManager.new({ :workers_number => workers_number.to_i, :queue => queue_name, :worker_options => { :timeout => workers_timeout.to_i } }).start
60
+ end
61
+
62
+ # stay alive
63
+ while(true)
64
+ sleep(5)
65
+ end
66
+ end
67
+
68
+ def dj_pid
69
+ return 'tmp/pids/delayed_job.pid'
70
+ end
71
+
72
+ task :start, [ :args_expr ] => :environment do |t, args|
73
+ puts "#{args[:args_expr]}"
74
+
75
+ cmd = %(if [ -f #{dj_pid} ] && [ -n `cat #{dj_pid}` ] && [ ps -p `cat #{dj_pid}` > /dev/null ]; then sudo kill -9 `cat #{dj_pid}`; fi
76
+ (bundle exec rake "delayed_job:run[#{args[:args_expr]}]" >> log/delayed_job.log 2>&1) & (echo $! > tmp/pids/dj.pid)
77
+ )
78
+
79
+ puts "executing: #{cmd}"
80
+
81
+ # execute cmd
82
+ %x(#{cmd})
83
+ end
84
+
85
+ task :stop => :environment do |t, args|
86
+ cmd = %(if [ -f #{dj_pid} ] && [ -n `cat #{dj_pid}` ] && [ ps -p `cat #{dj_pid}` > /dev/null ]; then kill -9 `cat #{dj_pid}`; fi)
87
+
88
+ # execute cmd
89
+ %x(#{cmd})
90
+ end
91
+
92
+ task :print_options, [ :args_expr ] => :environment do |t, args|
93
+ args.with_defaults(:args_expr => "name=abdo&fruits[]=bananas&fruits[]=dates")
94
+ options = Rack::Utils.parse_nested_query(args[:args_expr])
95
+
96
+ puts options
97
+ end
98
+
99
+ task :kill_all do
100
+ cmd = %(ps aux | grep delayed_job | awk -F" " '{ print $2 }' | xargs kill -9)
101
+ %x(#{cmd})
102
+ end
103
+ end
data/test/database.yml ADDED
@@ -0,0 +1,15 @@
1
+ mysql2:
2
+ adapter: mysql2
3
+ database: delayed_job_test
4
+ username: dj
5
+ encoding: utf8
6
+ pool: 300
7
+
8
+ postgresql:
9
+ adapter: postgresql
10
+ database: delayed_job_test
11
+ username: postgres
12
+
13
+ sqlite3:
14
+ adapter: sqlite3
15
+ database: ":memory:"
@@ -0,0 +1,47 @@
1
+ require "test_helper"
2
+ require "delayed_job_active_record_threaded"
3
+ require "rails"
4
+ require "rails/test_help"
5
+ require "minitest/rails"
6
+
7
+ class TestJob < Struct.new(:id)
8
+ def perform
9
+ story = Story.find(id)
10
+ sleep(rand(2))
11
+ puts "Telling #{story.text}\n" if story
12
+ end
13
+ end
14
+
15
+ class DelayedJobActiveRecordThreadedTest < ActiveSupport::TestCase #MiniTest::Unit::TestCase
16
+ self.use_transactional_fixtures = false
17
+
18
+ QUEUE_NAME = "StoriesQueue"
19
+
20
+ before do
21
+ #Delayed::Job.new(:run_at => 10.seconds.ago).save
22
+ 100.times { |i|
23
+ s = Story.create!(:text => "Story #{i}")
24
+ Delayed::Job.enqueue(TestJob.new(s.id), :queue => QUEUE_NAME, :priority => i, :delayable_id => s.id, :delayable_type => s.class.name, :run_at => 10.seconds.ago)
25
+ }
26
+ end
27
+
28
+ after do
29
+ end
30
+
31
+ # this is VERY dependent on the database pool size
32
+ test "should process jobs without crashing" do
33
+ mgr = Delayed::HomeManager.new({ :sleep_time => 1, :workers_number => 25, :queue => QUEUE_NAME })
34
+
35
+ puts "Number of jobs in queue is: #{Delayed::Job.count}"
36
+
37
+ mgr.start
38
+
39
+ sleep(10)
40
+
41
+ mgr.kill
42
+
43
+ puts "Number of threads: #{Thread.list.count}"
44
+
45
+ assert Delayed::Job.where("queue = ?", QUEUE_NAME).count == 0, "Expected to have an empty queue after 10 seconds. Queue size was #{Delayed::Job.count} instead"
46
+ end
47
+ end
@@ -0,0 +1,71 @@
1
+ require 'minitest/unit'
2
+ require 'minitest/autorun'
3
+ require 'turn'
4
+
5
+
6
+ require 'yaml'
7
+ require 'active_record'
8
+ require 'delayed_job'
9
+ require "#{File.dirname(__FILE__)}/../lib/delayed_job_active_record_threaded"
10
+ require "#{File.dirname(__FILE__)}/../lib/delayed/job"
11
+
12
+ # copied (and modified) from https://github.com/collectiveidea/delayed_job_active_record/blob/master/spec/helper.rb
13
+ begin
14
+ require 'protected_attributes'
15
+ rescue LoadError
16
+ end
17
+
18
+ $logger = Logger.new('/tmp/dj.log')
19
+ ENV['RAILS_ENV'] = 'test'
20
+
21
+ db_adapter, gemfile = ENV["ADAPTER"], ENV["BUNDLE_GEMFILE"]
22
+ db_adapter ||= gemfile && gemfile[%r(gemfiles/(.*?)/)] && $1
23
+ #db_adapter ||= 'sqlite3'
24
+ db_adapter ||= 'mysql2'
25
+
26
+ config = YAML.load(File.read('test/database.yml'))
27
+ ActiveRecord::Base.establish_connection config[db_adapter]
28
+ ActiveRecord::Base.logger = $logger
29
+ ActiveRecord::Migration.verbose = true
30
+
31
+ ActiveRecord::Schema.define do
32
+ create_table :delayed_jobs, :force => true do |table|
33
+ table.integer :priority, :default => 0
34
+ table.integer :attempts, :default => 0
35
+ table.text :handler
36
+ table.text :last_error
37
+ table.datetime :run_at
38
+ table.datetime :locked_at
39
+ table.datetime :failed_at
40
+ table.string :locked_by
41
+ table.string :queue
42
+ table.integer :delayable_id
43
+ table.string :delayable_type
44
+ table.timestamps
45
+ end
46
+
47
+ add_index :delayed_jobs, [:priority, :run_at], :name => 'delayed_jobs_priority'
48
+ add_index :delayed_jobs, [:delayable_id, :delayable_type], :name => 'delayed_jobs_delayable'
49
+
50
+ create_table :stories, :primary_key => :story_id, :force => true do |table|
51
+ table.string :text
52
+ table.boolean :scoped, :default => true
53
+ end
54
+ end
55
+
56
+ # Purely useful for test cases...
57
+ class Story < ActiveRecord::Base
58
+ if ::ActiveRecord::VERSION::MAJOR < 4 && ActiveRecord::VERSION::MINOR < 2
59
+ set_primary_key :story_id
60
+ else
61
+ self.primary_key = :story_id
62
+ end
63
+ def tell; text; end
64
+ def whatever(n, _); tell*n; end
65
+ default_scope { where(:scoped => true) }
66
+
67
+ handle_asynchronously :whatever
68
+ end
69
+
70
+ # Add this directory so the ActiveSupport autoloading works
71
+ ActiveSupport::Dependencies.autoload_paths << File.dirname(__FILE__)
metadata ADDED
@@ -0,0 +1,213 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: delayed_job_active_record_threaded
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Abdo Achkar
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2013-09-17 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ~>
18
+ - !ruby/object:Gem::Version
19
+ version: '1.3'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ~>
25
+ - !ruby/object:Gem::Version
26
+ version: '1.3'
27
+ - !ruby/object:Gem::Dependency
28
+ name: minitest
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ~>
32
+ - !ruby/object:Gem::Version
33
+ version: 4.7.3
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ~>
39
+ - !ruby/object:Gem::Version
40
+ version: 4.7.3
41
+ - !ruby/object:Gem::Dependency
42
+ name: rails
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '>='
46
+ - !ruby/object:Gem::Version
47
+ version: '3.0'
48
+ - - <
49
+ - !ruby/object:Gem::Version
50
+ version: '4.1'
51
+ type: :development
52
+ prerelease: false
53
+ version_requirements: !ruby/object:Gem::Requirement
54
+ requirements:
55
+ - - '>='
56
+ - !ruby/object:Gem::Version
57
+ version: '3.0'
58
+ - - <
59
+ - !ruby/object:Gem::Version
60
+ version: '4.1'
61
+ - !ruby/object:Gem::Dependency
62
+ name: turn
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - '>='
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - '>='
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: rake
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - '>='
80
+ - !ruby/object:Gem::Version
81
+ version: '0'
82
+ type: :development
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '>='
87
+ - !ruby/object:Gem::Version
88
+ version: '0'
89
+ - !ruby/object:Gem::Dependency
90
+ name: sqlite3-ruby
91
+ requirement: !ruby/object:Gem::Requirement
92
+ requirements:
93
+ - - '>='
94
+ - !ruby/object:Gem::Version
95
+ version: 1.3.1
96
+ type: :development
97
+ prerelease: false
98
+ version_requirements: !ruby/object:Gem::Requirement
99
+ requirements:
100
+ - - '>='
101
+ - !ruby/object:Gem::Version
102
+ version: 1.3.1
103
+ - !ruby/object:Gem::Dependency
104
+ name: activerecord
105
+ requirement: !ruby/object:Gem::Requirement
106
+ requirements:
107
+ - - '>='
108
+ - !ruby/object:Gem::Version
109
+ version: '3.0'
110
+ - - <
111
+ - !ruby/object:Gem::Version
112
+ version: '4.1'
113
+ type: :runtime
114
+ prerelease: false
115
+ version_requirements: !ruby/object:Gem::Requirement
116
+ requirements:
117
+ - - '>='
118
+ - !ruby/object:Gem::Version
119
+ version: '3.0'
120
+ - - <
121
+ - !ruby/object:Gem::Version
122
+ version: '4.1'
123
+ - !ruby/object:Gem::Dependency
124
+ name: delayed_job
125
+ requirement: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - '>='
128
+ - !ruby/object:Gem::Version
129
+ version: '3.0'
130
+ - - <
131
+ - !ruby/object:Gem::Version
132
+ version: '4.1'
133
+ type: :runtime
134
+ prerelease: false
135
+ version_requirements: !ruby/object:Gem::Requirement
136
+ requirements:
137
+ - - '>='
138
+ - !ruby/object:Gem::Version
139
+ version: '3.0'
140
+ - - <
141
+ - !ruby/object:Gem::Version
142
+ version: '4.1'
143
+ - !ruby/object:Gem::Dependency
144
+ name: celluloid
145
+ requirement: !ruby/object:Gem::Requirement
146
+ requirements:
147
+ - - '>='
148
+ - !ruby/object:Gem::Version
149
+ version: 0.14.1
150
+ type: :runtime
151
+ prerelease: false
152
+ version_requirements: !ruby/object:Gem::Requirement
153
+ requirements:
154
+ - - '>='
155
+ - !ruby/object:Gem::Version
156
+ version: 0.14.1
157
+ description: Allows going through delayed job queues using threads instead of processes
158
+ email:
159
+ - abdo.achkar@gmail.com
160
+ executables: []
161
+ extensions: []
162
+ extra_rdoc_files: []
163
+ files:
164
+ - .gitignore
165
+ - .travis.yml
166
+ - Gemfile
167
+ - LICENSE.txt
168
+ - README.md
169
+ - Rakefile
170
+ - delayed_job_active_record_threaded.gemspec
171
+ - lib/delayed/fake_job.rb
172
+ - lib/delayed/home_manager.rb
173
+ - lib/delayed/home_worker.rb
174
+ - lib/delayed/job.rb
175
+ - lib/delayed/railtie.rb
176
+ - lib/delayed_job_active_record_threaded.rb
177
+ - lib/generators/delayed_job/active_record_generator.rb
178
+ - lib/generators/delayed_job/next_migration_version.rb
179
+ - lib/generators/delayed_job/templates/migration.rb
180
+ - lib/generators/delayed_job/templates/upgrade_migration.rb
181
+ - lib/generators/delayed_job/upgrade_generator.rb
182
+ - lib/tasks/dj.rake
183
+ - test/database.yml
184
+ - test/delayed_job_active_record_threaded_test.rb
185
+ - test/test_helper.rb
186
+ homepage: ''
187
+ licenses:
188
+ - MIT
189
+ metadata: {}
190
+ post_install_message:
191
+ rdoc_options: []
192
+ require_paths:
193
+ - lib
194
+ required_ruby_version: !ruby/object:Gem::Requirement
195
+ requirements:
196
+ - - '>='
197
+ - !ruby/object:Gem::Version
198
+ version: '0'
199
+ required_rubygems_version: !ruby/object:Gem::Requirement
200
+ requirements:
201
+ - - '>='
202
+ - !ruby/object:Gem::Version
203
+ version: '0'
204
+ requirements: []
205
+ rubyforge_project:
206
+ rubygems_version: 2.0.3
207
+ signing_key:
208
+ specification_version: 4
209
+ summary: Process the delayed job queue using threads.
210
+ test_files:
211
+ - test/database.yml
212
+ - test/delayed_job_active_record_threaded_test.rb
213
+ - test/test_helper.rb