spx-resque-multi-job-forks 0.3.2
Sign up to get free protection for your applications and to get access to all the features.
- data/lib/resque-multi-job-forks.rb +88 -0
- data/test/helper.rb +60 -0
- data/test/test_resque-multi-job-forks.rb +58 -0
- metadata +117 -0
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'resque'
|
2
|
+
require 'resque/worker'
|
3
|
+
|
4
|
+
module Resque
|
5
|
+
class Worker
|
6
|
+
attr_accessor :seconds_per_fork
|
7
|
+
attr_accessor :jobs_per_fork
|
8
|
+
attr_reader :jobs_processed
|
9
|
+
|
10
|
+
unless method_defined?(:shutdown_without_multi_job_forks)
|
11
|
+
def perform_with_multi_job_forks(job = nil)
|
12
|
+
perform_without_multi_job_forks(job)
|
13
|
+
hijack_fork unless fork_hijacked?
|
14
|
+
@jobs_processed += 1
|
15
|
+
end
|
16
|
+
alias_method :perform_without_multi_job_forks, :perform
|
17
|
+
alias_method :perform, :perform_with_multi_job_forks
|
18
|
+
|
19
|
+
def shutdown_with_multi_job_forks
|
20
|
+
release_fork if fork_hijacked? && fork_job_limit_reached?
|
21
|
+
shutdown_without_multi_job_forks
|
22
|
+
end
|
23
|
+
alias_method :shutdown_without_multi_job_forks, :shutdown?
|
24
|
+
alias_method :shutdown?, :shutdown_with_multi_job_forks
|
25
|
+
end
|
26
|
+
|
27
|
+
def fork_hijacked?
|
28
|
+
@release_fork_limit
|
29
|
+
end
|
30
|
+
|
31
|
+
def hijack_fork
|
32
|
+
log 'hijack fork.'
|
33
|
+
@suppressed_fork_hooks = [Resque.after_fork, Resque.before_fork]
|
34
|
+
Resque.after_fork = Resque.before_fork = nil
|
35
|
+
@release_fork_limit = fork_job_limit
|
36
|
+
@jobs_processed = 0
|
37
|
+
@cant_fork = true
|
38
|
+
end
|
39
|
+
|
40
|
+
def release_fork
|
41
|
+
log "jobs processed by child: #{jobs_processed}"
|
42
|
+
run_hook :before_child_exit, self
|
43
|
+
Resque.after_fork, Resque.before_fork = *@suppressed_fork_hooks
|
44
|
+
@release_fork_limit = @jobs_processed = @cant_fork = nil
|
45
|
+
log 'hijack over, counter terrorists win.'
|
46
|
+
@shutdown = true unless $TESTING
|
47
|
+
end
|
48
|
+
|
49
|
+
def fork_job_limit
|
50
|
+
jobs_per_fork.nil? ? Time.now.to_i + seconds_per_fork : jobs_per_fork
|
51
|
+
end
|
52
|
+
|
53
|
+
def fork_job_limit_reached?
|
54
|
+
fork_job_limit_remaining <= 0 ? true : false
|
55
|
+
end
|
56
|
+
|
57
|
+
def fork_job_limit_remaining
|
58
|
+
jobs_per_fork.nil? ? @release_fork_limit - Time.now.to_i : jobs_per_fork - @jobs_processed
|
59
|
+
end
|
60
|
+
|
61
|
+
def seconds_per_fork
|
62
|
+
@seconds_per_fork ||= minutes_per_fork * 60
|
63
|
+
end
|
64
|
+
|
65
|
+
def minutes_per_fork
|
66
|
+
ENV['MINUTES_PER_FORK'].nil? ? 1 : ENV['MINUTES_PER_FORK'].to_i
|
67
|
+
end
|
68
|
+
|
69
|
+
def jobs_per_fork
|
70
|
+
@jobs_per_fork ||= ENV['JOBS_PER_FORK'].nil? ? nil : ENV['JOBS_PER_FORK'].to_i
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
# the `before_child_exit` hook will run in the child process
|
75
|
+
# right before the child process terminates
|
76
|
+
#
|
77
|
+
# Call with a block to set the hook.
|
78
|
+
# Call with no arguments to return the hook.
|
79
|
+
def self.before_child_exit(&block)
|
80
|
+
block ? (@before_child_exit = block) : @before_child_exit
|
81
|
+
end
|
82
|
+
|
83
|
+
# Set the before_child_exit proc.
|
84
|
+
def self.before_child_exit=(before_child_exit)
|
85
|
+
@before_child_exit = before_child_exit
|
86
|
+
end
|
87
|
+
|
88
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
2
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
3
|
+
|
4
|
+
$TESTING = true
|
5
|
+
|
6
|
+
require 'rubygems'
|
7
|
+
require 'bundler'
|
8
|
+
Bundler.setup
|
9
|
+
Bundler.require
|
10
|
+
require 'test/unit'
|
11
|
+
require 'resque-multi-job-forks'
|
12
|
+
|
13
|
+
# setup redis & resque.
|
14
|
+
redis = Redis.new(:db => 1)
|
15
|
+
Resque.redis = redis
|
16
|
+
|
17
|
+
# adds simple STDOUT logging to test workers.
|
18
|
+
# set `VERBOSE=true` when running the tests to view resques log output.
|
19
|
+
module Resque
|
20
|
+
class Worker
|
21
|
+
def log(msg)
|
22
|
+
puts "*** #{msg}" unless ENV['VERBOSE'].nil?
|
23
|
+
end
|
24
|
+
alias_method :log!, :log
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# stores a record of the job processing sequence.
|
29
|
+
# you may wish to reset this in the test `setup` method.
|
30
|
+
$SEQUENCE = []
|
31
|
+
|
32
|
+
# test job, tracks sequence.
|
33
|
+
class SequenceJob
|
34
|
+
@queue = :jobs
|
35
|
+
def self.perform(i)
|
36
|
+
$SEQUENCE << "work_#{i}".to_sym
|
37
|
+
sleep(2)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
class QuickSequenceJob
|
42
|
+
@queue = :jobs
|
43
|
+
def self.perform(i)
|
44
|
+
$SEQUENCE << "work_#{i}".to_sym
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
|
49
|
+
# test hooks, tracks sequence.
|
50
|
+
Resque.after_fork do
|
51
|
+
$SEQUENCE << :after_fork
|
52
|
+
end
|
53
|
+
|
54
|
+
Resque.before_fork do
|
55
|
+
$SEQUENCE << :before_fork
|
56
|
+
end
|
57
|
+
|
58
|
+
Resque.before_child_exit do |worker|
|
59
|
+
$SEQUENCE << "before_child_exit_#{worker.jobs_processed}".to_sym
|
60
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/helper'
|
2
|
+
|
3
|
+
class TestResqueMultiJobForks < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
$SEQUENCE = []
|
6
|
+
Resque.redis.flushdb
|
7
|
+
@worker = Resque::Worker.new(:jobs)
|
8
|
+
end
|
9
|
+
|
10
|
+
def test_timeout_limit_sequence_of_events
|
11
|
+
# only allow enough time for 3 jobs to process.
|
12
|
+
@worker.seconds_per_fork = 3
|
13
|
+
|
14
|
+
Resque.enqueue(SequenceJob, 1)
|
15
|
+
Resque.enqueue(SequenceJob, 2)
|
16
|
+
Resque.enqueue(SequenceJob, 3)
|
17
|
+
Resque.enqueue(SequenceJob, 4)
|
18
|
+
|
19
|
+
# make sure we don't take longer then 15 seconds.
|
20
|
+
begin
|
21
|
+
Timeout::timeout(15) { @worker.work(1) }
|
22
|
+
rescue Timeout::Error
|
23
|
+
end
|
24
|
+
|
25
|
+
# test the sequence is correct.
|
26
|
+
assert_equal([:before_fork, :after_fork, :work_1, :work_2, :work_3,
|
27
|
+
:before_child_exit_3, :before_fork, :after_fork, :work_4,
|
28
|
+
:before_child_exit_1], $SEQUENCE, 'correct sequence')
|
29
|
+
end
|
30
|
+
|
31
|
+
# test we can also limit fork job process by a job limit.
|
32
|
+
def test_job_limit_sequence_of_events
|
33
|
+
# only allow enough time for 3 jobs to process.
|
34
|
+
ENV['JOBS_PER_FORK'] = '20'
|
35
|
+
|
36
|
+
# queue 40 jobs.
|
37
|
+
(1..40).each { |i| Resque.enqueue(QuickSequenceJob, i) }
|
38
|
+
|
39
|
+
begin
|
40
|
+
Timeout::timeout(3) { @worker.work(1) }
|
41
|
+
rescue Timeout::Error
|
42
|
+
end
|
43
|
+
|
44
|
+
assert_equal :before_fork, $SEQUENCE[0], 'first before_fork call.'
|
45
|
+
assert_equal :after_fork, $SEQUENCE[1], 'first after_fork call.'
|
46
|
+
assert_equal :work_20, $SEQUENCE[21], '20th chunk of work.'
|
47
|
+
assert_equal :before_child_exit_20, $SEQUENCE[22], 'first before_child_exit call.'
|
48
|
+
assert_equal :before_fork, $SEQUENCE[23], 'final before_fork call.'
|
49
|
+
assert_equal :after_fork, $SEQUENCE[24], 'final after_fork call.'
|
50
|
+
assert_equal :work_40, $SEQUENCE[44], '40th chunk of work.'
|
51
|
+
assert_equal :before_child_exit_20, $SEQUENCE[45], 'final before_child_exit call.'
|
52
|
+
end
|
53
|
+
|
54
|
+
def teardown
|
55
|
+
# make sure we don't clobber any other tests.
|
56
|
+
ENV['JOBS_PER_FORK'] = nil
|
57
|
+
end
|
58
|
+
end
|
metadata
ADDED
@@ -0,0 +1,117 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: spx-resque-multi-job-forks
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.3.2
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Mick Staugaard
|
9
|
+
- Luke Antins
|
10
|
+
autorequire:
|
11
|
+
bindir: bin
|
12
|
+
cert_chain: []
|
13
|
+
date: 2012-06-28 00:00:00.000000000 Z
|
14
|
+
dependencies:
|
15
|
+
- !ruby/object:Gem::Dependency
|
16
|
+
name: resque
|
17
|
+
requirement: !ruby/object:Gem::Requirement
|
18
|
+
none: false
|
19
|
+
requirements:
|
20
|
+
- - ~>
|
21
|
+
- !ruby/object:Gem::Version
|
22
|
+
version: 1.20.0
|
23
|
+
type: :runtime
|
24
|
+
prerelease: false
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
26
|
+
none: false
|
27
|
+
requirements:
|
28
|
+
- - ~>
|
29
|
+
- !ruby/object:Gem::Version
|
30
|
+
version: 1.20.0
|
31
|
+
- !ruby/object:Gem::Dependency
|
32
|
+
name: json
|
33
|
+
requirement: !ruby/object:Gem::Requirement
|
34
|
+
none: false
|
35
|
+
requirements:
|
36
|
+
- - ! '>='
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
type: :runtime
|
40
|
+
prerelease: false
|
41
|
+
version_requirements: !ruby/object:Gem::Requirement
|
42
|
+
none: false
|
43
|
+
requirements:
|
44
|
+
- - ! '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: rake
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :development
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: !ruby/object:Gem::Requirement
|
58
|
+
none: false
|
59
|
+
requirements:
|
60
|
+
- - ! '>='
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: bundler
|
65
|
+
requirement: !ruby/object:Gem::Requirement
|
66
|
+
none: false
|
67
|
+
requirements:
|
68
|
+
- - ! '>='
|
69
|
+
- !ruby/object:Gem::Version
|
70
|
+
version: '0'
|
71
|
+
type: :development
|
72
|
+
prerelease: false
|
73
|
+
version_requirements: !ruby/object:Gem::Requirement
|
74
|
+
none: false
|
75
|
+
requirements:
|
76
|
+
- - ! '>='
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '0'
|
79
|
+
description: When your resque jobs are frequent and fast, the overhead of forking
|
80
|
+
and running your after_fork might get too big.
|
81
|
+
email:
|
82
|
+
- mick@zendesk.com
|
83
|
+
- luke@lividpenguin.com
|
84
|
+
executables: []
|
85
|
+
extensions: []
|
86
|
+
extra_rdoc_files: []
|
87
|
+
files:
|
88
|
+
- lib/resque-multi-job-forks.rb
|
89
|
+
- test/helper.rb
|
90
|
+
- test/test_resque-multi-job-forks.rb
|
91
|
+
homepage: http://github.com/staugaard/resque-multi-job-forks
|
92
|
+
licenses: []
|
93
|
+
post_install_message:
|
94
|
+
rdoc_options: []
|
95
|
+
require_paths:
|
96
|
+
- lib
|
97
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
98
|
+
none: false
|
99
|
+
requirements:
|
100
|
+
- - ! '>='
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
104
|
+
none: false
|
105
|
+
requirements:
|
106
|
+
- - ! '>='
|
107
|
+
- !ruby/object:Gem::Version
|
108
|
+
version: '0'
|
109
|
+
requirements: []
|
110
|
+
rubyforge_project:
|
111
|
+
rubygems_version: 1.8.24
|
112
|
+
signing_key:
|
113
|
+
specification_version: 3
|
114
|
+
summary: Have your resque workers process more that one job
|
115
|
+
test_files:
|
116
|
+
- test/helper.rb
|
117
|
+
- test/test_resque-multi-job-forks.rb
|