rescheduler 0.4.1 → 0.5.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/bin/rescheduler_launch +50 -0
- data/lib/rescheduler/sync.rb +44 -0
- data/lib/rescheduler/worker.rb +293 -0
- data/lib/rescheduler.rb +350 -271
- metadata +16 -19
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 5e0575537baa956f4e8628a7892d9bcc0c3c6a80
|
4
|
+
data.tar.gz: 1481a1e1b3f2f4e491787793466db8479235ff61
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 2107fe553f4b8a9570b8623d5d46b46552d8ead112201c7d80dd62df309ad280b3fb9d3c3876995ac02674a035e02f6d5f2653e959b03b346fdf589e2600c26a
|
7
|
+
data.tar.gz: f897e041abbab34c119c70aad94c9333c4f4c94b52d72940b72483844d2be4f38eea31de2d38bc91401c6e2f7c9e3083db0b2b1b014926f1d383a7f5c715a844
|
@@ -0,0 +1,50 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
=begin
|
3
|
+
rescheduler_launch <options> [<worker_name> <jobs.rb>]
|
4
|
+
Options:
|
5
|
+
--rails=<folder> Use rails environment from the givne folder
|
6
|
+
--log=<file> Specify logfile (in Rails mode default to <rails_foler>/log/<worker_name>.log)
|
7
|
+
--env=<env> Set these ENV into the new process
|
8
|
+
--debug Do not daemonize or redirect output, work with current STDOUT
|
9
|
+
--respawn_jobs=N Respawn worker after N jobs
|
10
|
+
--respawn_time=N Respawn every N seconds (NOTE: Only will respawn after finish a job)
|
11
|
+
Internal purpose only:
|
12
|
+
--respawn Set to indicate this is a respawn of the named worker and will take over the
|
13
|
+
name without deleting stats
|
14
|
+
=end
|
15
|
+
|
16
|
+
# First parse options
|
17
|
+
opts = {}
|
18
|
+
while ARGV.size > 0 && ARGV[0][0..1] == '--'
|
19
|
+
opt,val = ARGV.shift[2..-1].split('=',2)
|
20
|
+
val ||= true
|
21
|
+
opts[opt] = val
|
22
|
+
end
|
23
|
+
|
24
|
+
opts['worker_name'] = ARGV.shift
|
25
|
+
opts['worker_file'] = ARGV.shift
|
26
|
+
|
27
|
+
abort "rescheduler_launch <options> <worker_name> <jobs.rb>" unless opts['worker_file'] && File.exists?(opts['worker_file'])
|
28
|
+
|
29
|
+
# Daemonize first before we load rails
|
30
|
+
begin
|
31
|
+
Process.daemon unless opts['debug'] # Do not daemonize if debugging
|
32
|
+
rescue NotImplementedError
|
33
|
+
# This happens in windows, it is OK
|
34
|
+
end
|
35
|
+
|
36
|
+
# Load rails environment if --rails is specified
|
37
|
+
require File.join(opts['rails'], 'config/environment.rb') if opts['rails']
|
38
|
+
|
39
|
+
# Load rescheduler
|
40
|
+
require File.expand_path('../../lib/rescheduler', __FILE__)
|
41
|
+
|
42
|
+
Rescheduler::Worker.register(opts)
|
43
|
+
Rescheduler::Worker.redirect_logging(opts) unless opts['debug']
|
44
|
+
|
45
|
+
# Running the job file
|
46
|
+
begin
|
47
|
+
Kernel.load opts['worker_file']
|
48
|
+
ensure
|
49
|
+
Rescheduler::Worker.unregister unless Rescheduler::Worker.respawn_if_requested
|
50
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
module Rescheduler
|
2
|
+
module Sync
|
3
|
+
extend self
|
4
|
+
|
5
|
+
def lock(name, opts={})
|
6
|
+
raise "Requires a block to be supplied" unless block_given? # Maybe later offer naked lock/unlocks?
|
7
|
+
raise "Need a valid string name" unless name.is_a?(String)
|
8
|
+
|
9
|
+
res = do_lock(name, opts) # This would block
|
10
|
+
return nil unless res # Timeout or failed somehow
|
11
|
+
|
12
|
+
begin
|
13
|
+
yield
|
14
|
+
ensure
|
15
|
+
do_unlock(name)
|
16
|
+
end
|
17
|
+
return true # Lock was successful
|
18
|
+
end
|
19
|
+
|
20
|
+
# Forcefully unlock and delete existing locks
|
21
|
+
def clear!(name)
|
22
|
+
redis.del(rk_exists_name(name), rk_lock_name(name))
|
23
|
+
end
|
24
|
+
|
25
|
+
private def do_lock(name, opts)
|
26
|
+
# Make sure semaphore for a given name is only created once
|
27
|
+
if redis.getset(rk_exists_name(name), 1)
|
28
|
+
# Already created, block wait for the release (possibility for unlock)
|
29
|
+
return redis.brpop(rk_lock_name(name), timeout: (opts[:timeout] || 0))
|
30
|
+
else
|
31
|
+
# First time, get the lock automatically
|
32
|
+
return true
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
private def do_unlock(name)
|
37
|
+
redis.lpush(rk_lock_name(name), 1)
|
38
|
+
end
|
39
|
+
|
40
|
+
private def redis; Rescheduler.send(:redis); end
|
41
|
+
private def rk_exists_name(name); return "#{Rescheduler.prefix}Sync:#{name}"; end
|
42
|
+
private def rk_lock_name(name); return "#{Rescheduler.prefix}SyncQ:#{name}"; end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,293 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module Rescheduler
|
4
|
+
module Worker
|
5
|
+
extend self
|
6
|
+
# Each worker is a process of it own, handled by this singleton module
|
7
|
+
|
8
|
+
NONCMD_OPTS = %w[pid machine worker_name worker_file]
|
9
|
+
CMD_OPTS = %w[rails log env respawn] # respanw is for internal only
|
10
|
+
attr_accessor :launch_options
|
11
|
+
def worker_name; @launch_options && @launch_options['worker_name']; end
|
12
|
+
def worker_index; @launch_options && @launch_options['worker_index'] || -1; end
|
13
|
+
# ====================================================================
|
14
|
+
# Controller
|
15
|
+
# ====================================================================
|
16
|
+
WORKERCMD_SUSPEND = 'suspend'
|
17
|
+
WORKERCMD_RESUME = 'resume'
|
18
|
+
WORKERCMD_STOP = 'stop'
|
19
|
+
WORKERCMD_RESTART = 'restart'
|
20
|
+
[WORKERCMD_SUSPEND, WORKERCMD_RESUME, WORKERCMD_STOP, WORKERCMD_RESTART].each do |cmd|
|
21
|
+
define_method(cmd) {|name| redis.lpush(rk_queue(name), cmd) }
|
22
|
+
define_method(cmd + '_all') {|pattern| cmd_to_pattern(pattern, cmd) }
|
23
|
+
end
|
24
|
+
|
25
|
+
def workers_from_pattern(pattern)
|
26
|
+
pattern = pattern.gsub('%', '*')
|
27
|
+
workers = redis.keys(rk_worker(pattern))
|
28
|
+
kl = rk_worker('').length
|
29
|
+
workers.map {|w| w[kl..-1]}
|
30
|
+
end
|
31
|
+
|
32
|
+
def cmd_to_pattern(pattern, cmd)
|
33
|
+
ws = []
|
34
|
+
workers_from_pattern(pattern).each do |wname|
|
35
|
+
redis.lpush(rk_queue(wname), cmd)
|
36
|
+
ws << wname
|
37
|
+
end
|
38
|
+
return ws
|
39
|
+
end
|
40
|
+
|
41
|
+
def kill_all(pattern, force=false)
|
42
|
+
workers_from_pattern(pattern).each do |wname|
|
43
|
+
kill(wname, force)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
# Do not run this unless everything else fails, it kills unconditionally
|
48
|
+
def kill(worker_name, force=false)
|
49
|
+
key = rk_worker(worker_name)
|
50
|
+
opts = redis.hgetall(key)
|
51
|
+
if force || opts['machine'] == Socket.gethostname
|
52
|
+
stop(worker_name)
|
53
|
+
sleep 1 # Give the process a second to quick peacefully
|
54
|
+
redis.del(key)
|
55
|
+
# Actually kill the process
|
56
|
+
begin
|
57
|
+
pid = opts['pid']
|
58
|
+
# "HUP" does not get recognized in windows
|
59
|
+
Process.kill(9, pid.to_i) if pid && opts['machine'] == Socket.gethostname
|
60
|
+
rescue Errno::ESRCH, Errno::EPERM # No such process, Not permitted, these will be ginored
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def clean_dead_workers
|
66
|
+
hostname = Socket.gethostname # Only can cleanup PIDs on own machine
|
67
|
+
workers = redis.keys(rk_worker('*'))
|
68
|
+
workers.each do |w|
|
69
|
+
opts = redis.hgetall(w)
|
70
|
+
next unless opts['machine'] == hostname
|
71
|
+
next if pid_exists?(opts['pid'])
|
72
|
+
print "Cleaning dead worker: #{w.split(':',3).last}\n"
|
73
|
+
redis.del(w) # Remove the record (Main part of the cleanup)
|
74
|
+
end
|
75
|
+
nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def restart_self
|
79
|
+
Rescheduler.log_debug "[#{worker_name}] Restarting"
|
80
|
+
@respawn = true
|
81
|
+
Rescheduler.end_job_loop
|
82
|
+
@in_suspend = false
|
83
|
+
end
|
84
|
+
|
85
|
+
def handle_command(command)
|
86
|
+
case command
|
87
|
+
when WORKERCMD_SUSPEND
|
88
|
+
Rescheduler.log_debug "[#{worker_name}] Suspending"
|
89
|
+
@in_suspend = true
|
90
|
+
suspend_loop
|
91
|
+
when WORKERCMD_RESUME
|
92
|
+
if @in_suspend
|
93
|
+
Rescheduler.log_debug "[#{worker_name}] Resuming"
|
94
|
+
@in_suspend = false # This will resume the normal job loop
|
95
|
+
else
|
96
|
+
Rescheduler.log_debug "[#{worker_name}] Resume command received when not suspended"
|
97
|
+
end
|
98
|
+
when WORKERCMD_RESTART
|
99
|
+
restart_self
|
100
|
+
when WORKERCMD_STOP
|
101
|
+
Rescheduler.log_debug "[#{worker_name}] Stopping"
|
102
|
+
Rescheduler.end_job_loop
|
103
|
+
@in_suspend = false
|
104
|
+
else
|
105
|
+
Rescheduler.log_debug "[#{worker_name}] Unknown command: #{command}"
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def suspend_loop
|
110
|
+
while @in_suspend do
|
111
|
+
result = redis.brpop(rk_queue)
|
112
|
+
if result
|
113
|
+
command = result[1]
|
114
|
+
handle_command(command)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
# ====================================================================
|
120
|
+
# Runner
|
121
|
+
# ====================================================================
|
122
|
+
|
123
|
+
DEVNULL = '/dev/null'
|
124
|
+
def redirect_logging(opts)
|
125
|
+
return if windows_env? # Do not redirect in windows environment, let it run with cmd console
|
126
|
+
|
127
|
+
logfile = opts['log']
|
128
|
+
logfile ||= File.join(opts['rails'], "log/#{worker_name}.log") if opts['rails']
|
129
|
+
logfile ||= DEVNULL
|
130
|
+
|
131
|
+
# Redirect io
|
132
|
+
unless logfile == DEVNULL
|
133
|
+
# Code inspired by Daemonize::redirect_io
|
134
|
+
begin
|
135
|
+
STDOUT.reopen logfile, "a"
|
136
|
+
File.chmod(0644, logfile)
|
137
|
+
STDOUT.sync = true
|
138
|
+
rescue ::Exception
|
139
|
+
begin; STDOUT.reopen DEVNULL; rescue ::Exception; end
|
140
|
+
end
|
141
|
+
begin; STDERR.reopen STDOUT; rescue ::Exception; end
|
142
|
+
STDERR.sync = true
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
146
|
+
def is_respawning?; @launch_options && @launch_options['respawn']; end
|
147
|
+
|
148
|
+
def register(opt)
|
149
|
+
@launch_options = opt # Save for respawning
|
150
|
+
name_pattern = opt['worker_name']
|
151
|
+
|
152
|
+
last_wname = nil
|
153
|
+
widx = 0 # We do not preserve worker_index upon respawn
|
154
|
+
wname = nil
|
155
|
+
if opt['respawn']
|
156
|
+
wname = name_pattern
|
157
|
+
# Reset created for respawn purposes.
|
158
|
+
redis.hset(rk_worker(wname), 'created', Time.now.to_i)
|
159
|
+
else
|
160
|
+
loop do
|
161
|
+
wname = name_pattern.gsub('%', widx.to_s)
|
162
|
+
if wname == last_wname # For the name without %, we add one to the end if clashes
|
163
|
+
wname += widx.to_s
|
164
|
+
name_pattern += '%'
|
165
|
+
end
|
166
|
+
|
167
|
+
break if redis.hsetnx(rk_worker(wname), 'created', Time.now.to_i)
|
168
|
+
widx += 1
|
169
|
+
last_wname = wname
|
170
|
+
end
|
171
|
+
end
|
172
|
+
@launch_options['worker_name'] = wname # Save this in launch options
|
173
|
+
@launch_options['worker_index'] = widx # Sequence of the same worker
|
174
|
+
@launch_options['pid'] = Process.pid
|
175
|
+
@launch_options['machine'] = Socket.gethostname
|
176
|
+
|
177
|
+
# Setup the worker stats
|
178
|
+
redis.multi do
|
179
|
+
redis.hincrby(rk_worker, 'spawn_count', 1)
|
180
|
+
redis.hset(rk_worker, 'job_count', 0) # Reset the job count since respawn
|
181
|
+
redis.mapped_hmset(rk_worker, @launch_options) # Save launch options
|
182
|
+
redis.del(rk_queue) # Clear old control commands
|
183
|
+
end
|
184
|
+
nil
|
185
|
+
end
|
186
|
+
|
187
|
+
def unregister
|
188
|
+
redis.del(rk_worker)
|
189
|
+
end
|
190
|
+
|
191
|
+
def exists?(name)
|
192
|
+
redis.hexists(rk_workers, name)
|
193
|
+
end
|
194
|
+
|
195
|
+
def rk_worker(name = nil)
|
196
|
+
name ||= worker_name
|
197
|
+
Rescheduler.prefix + 'TMWORKER:' + name
|
198
|
+
end
|
199
|
+
|
200
|
+
def rk_queue(name = nil)
|
201
|
+
name ||= worker_name
|
202
|
+
Rescheduler.prefix + 'TMWORKERQUEUE:' + name
|
203
|
+
end
|
204
|
+
|
205
|
+
def stats
|
206
|
+
stats = {}
|
207
|
+
workers = redis.keys(rk_worker('*'))
|
208
|
+
kl = rk_worker('').length
|
209
|
+
workers.each do |w|
|
210
|
+
stats[w[kl..-1]] = redis.hgetall(w)
|
211
|
+
end
|
212
|
+
return stats
|
213
|
+
end
|
214
|
+
|
215
|
+
def named?
|
216
|
+
@launch_options && @launch_options.include?('worker_name')
|
217
|
+
end
|
218
|
+
|
219
|
+
def inc_job_count
|
220
|
+
# Only do this if we are launched as a worker
|
221
|
+
if named?
|
222
|
+
jc = redis.hincrby(rk_worker, 'job_count', 1)
|
223
|
+
# Check for respawn_jobs
|
224
|
+
respawn_jobs = @launch_options['respawn_jobs'].to_i
|
225
|
+
restart_self if respawn_jobs > 0 && jc >= respawn_jobs
|
226
|
+
# Check for respawn_time
|
227
|
+
respawn_time = @launch_options['respawn_time'].to_i
|
228
|
+
if respawn_time > 0
|
229
|
+
created = redis.hget(rk_worker, 'created')
|
230
|
+
restart_self if created && created.to_i + respawn_time < Time.now.to_i
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end
|
234
|
+
|
235
|
+
def respawn_if_requested
|
236
|
+
return false unless @respawn
|
237
|
+
@launch_options['respawn'] = true
|
238
|
+
spawn(@launch_options)
|
239
|
+
return true
|
240
|
+
end
|
241
|
+
|
242
|
+
def spawn(opts)
|
243
|
+
system_options = {}
|
244
|
+
env = load_env(opts['env']) if opts.include?('env')
|
245
|
+
env ||= {}
|
246
|
+
#system_options['chdir'] = opts['chdir'] if opts.include?('chdir')
|
247
|
+
|
248
|
+
cmd = "rescheduler_launch #{opt_to_str(opts)}"
|
249
|
+
print "EXEC: #{cmd}\n"
|
250
|
+
pid = Kernel.spawn(env, cmd, system_options)
|
251
|
+
Process.detach(pid)
|
252
|
+
end
|
253
|
+
|
254
|
+
def load_env(env)
|
255
|
+
env.is_a?(String) ? Hash[env.split(';').map{|e| a,b=e.split('=', 2); b ||= true; [a,b]}] : env
|
256
|
+
end
|
257
|
+
|
258
|
+
def pack_env(env)
|
259
|
+
return env if env.is_a?(String)
|
260
|
+
env.map {|k,v| "#{k}=#{v}"}.join(';')
|
261
|
+
end
|
262
|
+
|
263
|
+
def opt_to_str(opts)
|
264
|
+
name = opts['worker_name']
|
265
|
+
file = opts['worker_file']
|
266
|
+
args = opts.map do |k,v|
|
267
|
+
next if NONCMD_OPTS.include?(k)
|
268
|
+
if k == 'env'
|
269
|
+
"--env=#{pack_env(v)}"
|
270
|
+
elsif v==true || v == 'true'
|
271
|
+
"--#{k}"
|
272
|
+
else
|
273
|
+
"--#{k}=#{v}"
|
274
|
+
end
|
275
|
+
end
|
276
|
+
args << name
|
277
|
+
args << file
|
278
|
+
return args.join(' ')
|
279
|
+
end
|
280
|
+
|
281
|
+
def redis
|
282
|
+
Rescheduler.send :redis # Call private method in Rescheduler module
|
283
|
+
end
|
284
|
+
|
285
|
+
def windows_env?; RUBY_PLATFORM.end_with?('mingw32'); end
|
286
|
+
def pid_exists?(pid)
|
287
|
+
Process.kill(0, pid.to_i)
|
288
|
+
return true
|
289
|
+
rescue Errno::ESRCH
|
290
|
+
return false
|
291
|
+
end
|
292
|
+
end
|
293
|
+
end
|
data/lib/rescheduler.rb
CHANGED
@@ -1,142 +1,207 @@
|
|
1
|
+
require 'date'
|
1
2
|
require 'time' # Needed for Time.parse
|
2
3
|
require 'multi_json'
|
3
4
|
require 'redis'
|
4
5
|
|
6
|
+
require File.expand_path('../rescheduler/worker', __FILE__)
|
7
|
+
require File.expand_path('../rescheduler/sync', __FILE__)
|
8
|
+
|
9
|
+
=begin
|
10
|
+
|
11
|
+
Immediate Queue: "TMTUBE:queue" - List of qnids
|
12
|
+
Deferred Tasks: "TMDEFERRED" - Sorted set of all tasks based on their due date
|
13
|
+
Task Args: "TMARGS:qnid" - JSON args of the task
|
14
|
+
Running Tasks: "TMRUNNING" - ??
|
15
|
+
|
16
|
+
Maintenane: "TMMAINT" - ?? An internal queue for maintenance jobs
|
17
|
+
|
18
|
+
Worker Semaphore: "TMWORKERLOCK" - Exclusion semaphore for worker maintenance
|
19
|
+
Auto-increment Id:"TMCOUNTER" - Global unique id generator
|
20
|
+
|
21
|
+
Worker Registry: "TMWORKERS" - Map of workerid=>worker info
|
22
|
+
=end
|
23
|
+
|
24
|
+
# NOTE: We use class variables instead of class instance variables so that
|
25
|
+
# "include Rescheduler" would work as intended for DSL definition
|
26
|
+
|
27
|
+
|
5
28
|
module Rescheduler
|
6
29
|
extend self
|
7
30
|
|
8
31
|
# Setup configuration
|
9
|
-
|
10
|
-
|
32
|
+
def config
|
33
|
+
@@config ||= { prefix:'' }
|
34
|
+
@@config
|
35
|
+
end
|
36
|
+
|
37
|
+
def config=(c); @@config = c; end
|
11
38
|
|
12
|
-
|
13
|
-
#
|
39
|
+
#====================================================================
|
40
|
+
# Global management / Query
|
41
|
+
#====================================================================
|
14
42
|
def prefix
|
15
|
-
return
|
43
|
+
return @@config[:prefix]
|
16
44
|
end
|
17
45
|
|
18
|
-
def reinitialize
|
46
|
+
def reinitialize
|
19
47
|
keys = %w{TMCOUNTER TMMAINT TMDEFERRED TMARGS TMRUNNING}.map {|p| prefix + p }
|
20
|
-
%w{TMTUBE:*}.each do |p|
|
48
|
+
%w{TMTUBE:* TMARGS:*}.each do |p|
|
21
49
|
keys += redis.keys(prefix + p)
|
22
50
|
end
|
23
51
|
redis.del(keys)
|
24
52
|
end
|
25
53
|
|
26
|
-
#
|
27
|
-
|
54
|
+
# Warning: Linear time operation (see #show_queue)
|
55
|
+
def delete_queue(queue)
|
56
|
+
entries = show_queue(queue)
|
57
|
+
return 0 if entries.blank?
|
58
|
+
|
59
|
+
entries.map do |entry|
|
60
|
+
idelete(get_qnid(queue, entry))
|
61
|
+
end.length
|
62
|
+
end
|
63
|
+
|
64
|
+
def fast_delete_immediate_queue(queue) # NOTE: only use this when there is no inserters around
|
65
|
+
argkeys = redis.keys(rk_args(get_qnid(queue, '*')))
|
66
|
+
redis.multi do
|
67
|
+
redis.del(argkeys)
|
68
|
+
redis.del(rk_queue(queue))
|
69
|
+
end
|
70
|
+
nil
|
71
|
+
end
|
72
|
+
|
73
|
+
# Return a hash of statistics
|
28
74
|
def stats
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
75
|
+
stats = {}
|
76
|
+
@@runners ||= {}
|
77
|
+
@@runners.keys.each {|queue| stats[queue] = {} } unless @@runners.blank?
|
78
|
+
|
79
|
+
# Discover all immediate queues
|
80
|
+
ql = rk_queue('').length
|
81
|
+
redis.keys(rk_queue('*')).each do |rkqueue|
|
82
|
+
queue = rkqueue[ql..-1]
|
83
|
+
stats[queue] ||= {}
|
84
|
+
stats[queue][:immediate] = queue_length(queue)
|
85
|
+
end
|
40
86
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
87
|
+
# Get all the deferred
|
88
|
+
deferred = redis.zrange(rk_deferred, 0, -1, :with_scores=>true)
|
89
|
+
deferred.each do |qnid, ts|
|
90
|
+
queue = qnid_to_queue(qnid)
|
91
|
+
stats[queue] ||= {}
|
92
|
+
stats[queue][:deferred] ||= 0
|
93
|
+
stats[queue][:deferred] += 1
|
94
|
+
stats[queue][:first] ||= ts # First is first
|
95
|
+
end
|
50
96
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
queue = qnid_to_queue(qnid)
|
55
|
-
stats[queue] ||= {}
|
56
|
-
stats[queue][:deferred] ||= 0
|
57
|
-
stats[queue][:deferred] += 1
|
58
|
-
stats[queue][:first] ||= ts # First is first
|
59
|
-
end
|
60
|
-
|
61
|
-
# Get all the immediate
|
62
|
-
qus = stats.keys
|
63
|
-
quls = redis.multi do
|
64
|
-
qus.each { |queue| redis.llen(rk_queue(queue)) }
|
65
|
-
end
|
97
|
+
# Get all the immediate
|
98
|
+
return {:jobs=>stats, :workers=>Worker.stats}
|
99
|
+
end
|
66
100
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
101
|
+
#----------------------------------------------
|
102
|
+
# Queue management
|
103
|
+
#----------------------------------------------
|
104
|
+
# Returns number of jobs waiting to be handled in a queue (all immediate jobs)
|
105
|
+
def queue_length(queue)
|
106
|
+
return redis.llen(rk_queue(queue))
|
107
|
+
end
|
71
108
|
|
72
|
-
|
73
|
-
|
74
|
-
|
109
|
+
# Reads a background job and returns its properties; returns nil if the job does not exist
|
110
|
+
# Takes :queue and :id as arguments
|
111
|
+
def peek(options)
|
112
|
+
qnid = get_qnid(options[:queue], options[:id])
|
113
|
+
optstr = redis.get(rk_args(qnid))
|
114
|
+
return nil unless optstr
|
115
|
+
sopt = MultiJson.load(optstr, :symbolize_keys => true)
|
116
|
+
sopt[:queue] = options[:queue]
|
117
|
+
sopt[:id] = options[:id]
|
118
|
+
return sopt
|
119
|
+
end
|
75
120
|
|
76
|
-
|
77
|
-
|
78
|
-
|
121
|
+
# Warning: Linear time operation, where n is the number if items in all the queues
|
122
|
+
def show_queue(queue)
|
123
|
+
qstr = ":#{queue}:"
|
124
|
+
# TODO: Use SCAN after upgrade to Redis 2.8
|
125
|
+
redis.keys(rk_args(get_qnid(queue, '*'))).map {|k| k.split(qstr, 2).last }
|
79
126
|
end
|
80
127
|
|
81
|
-
|
82
|
-
#
|
83
|
-
|
84
|
-
pending, running, deferred = redis.multi do
|
85
|
-
redis.hkeys(rk_args)
|
86
|
-
redis.hkeys(rk_running)
|
87
|
-
redis.zrange(rk_deferred, 0, -1)
|
88
|
-
end
|
128
|
+
#----------------------------------------------
|
129
|
+
# Task management
|
130
|
+
#----------------------------------------------
|
89
131
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
132
|
+
# Check existence of one task
|
133
|
+
def exists?(options)
|
134
|
+
raise ArgumentError, 'Can not test existence without :id' unless options.include?(:id)
|
135
|
+
qnid = get_qnid(options[:queue], options[:id])
|
136
|
+
return redis.exists(rk_args(qnid))
|
95
137
|
end
|
96
138
|
|
97
|
-
|
139
|
+
# Delete one task
|
140
|
+
def delete(options)
|
141
|
+
qnid = get_qnid(options[:queue], options[:id])
|
142
|
+
idelete(qnid)
|
143
|
+
end
|
144
|
+
|
145
|
+
#====================================================================
|
98
146
|
# Task producer routines
|
147
|
+
#====================================================================
|
99
148
|
# Add an immediate task to the queue
|
100
149
|
def enqueue(options=nil)
|
101
|
-
options
|
150
|
+
internal_enqueue(options, false)
|
151
|
+
end
|
152
|
+
|
153
|
+
def enqueue_to_top(options = nil)
|
154
|
+
internal_enqueue(options, true)
|
155
|
+
end
|
156
|
+
|
157
|
+
def internal_enqueue(options, push_to_top)
|
158
|
+
sopt = options ? options.dup : {}
|
159
|
+
queue = sopt[:queue] || '' # Default queue name is ''
|
160
|
+
has_id = sopt.include?(:id)
|
161
|
+
job_id = sopt[:id] || redis.incr(rk_counter) # Default random unique id
|
162
|
+
|
102
163
|
now = Time.now.to_i
|
103
164
|
|
104
|
-
# Error check
|
105
|
-
validate_queue_name(
|
106
|
-
validate_recurrance(
|
165
|
+
# Error check
|
166
|
+
validate_queue_name(queue)
|
167
|
+
validate_recurrance(sopt)
|
107
168
|
|
108
169
|
# Convert due_in to due_at
|
109
|
-
if
|
110
|
-
|
111
|
-
|
170
|
+
if sopt.include?(:due_in)
|
171
|
+
# log_debug 'Both due_in and due_at specified, favoring due_in' if sopt.include?(:due_at)
|
172
|
+
sopt[:due_at] = now + sopt[:due_in]
|
112
173
|
end
|
113
174
|
|
114
|
-
|
115
|
-
user_id = options.include?(:id)
|
116
|
-
unless user_id
|
117
|
-
options[:id] = redis.incr(rk_counter)
|
118
|
-
end
|
175
|
+
qnid = get_qnid(queue, job_id)
|
119
176
|
|
120
|
-
ts =
|
121
|
-
ts
|
122
|
-
|
123
|
-
|
177
|
+
ts = sopt[:due_at].to_i
|
178
|
+
if ts == 0 || ts < now # immediate
|
179
|
+
ts = now
|
180
|
+
sopt.delete(:due_at)
|
181
|
+
else
|
182
|
+
raise ArgumentError, 'Can not enqueue_to_top deferred jobs' if push_to_top
|
183
|
+
sopt[:due_at] = ts # Convert :due_at to integer timestamp to be reused in recurrance
|
184
|
+
end
|
124
185
|
|
125
186
|
# Encode and save args
|
126
187
|
redis.multi do # Transaction to enqueue the job and save args together
|
127
|
-
if
|
188
|
+
if has_id # Delete possible existing job if user set id
|
128
189
|
redis.zrem(rk_deferred, qnid)
|
129
|
-
redis.lrem(rk_queue(
|
190
|
+
redis.lrem(rk_queue(queue), 0, qnid) # This is going to be slow for long queues
|
130
191
|
end
|
131
192
|
|
132
|
-
# Save
|
133
|
-
redis.
|
193
|
+
# Save args even if it is empty (for existence checks)
|
194
|
+
redis.set(rk_args(qnid), MultiJson.dump(sopt))
|
134
195
|
|
135
196
|
# Determine the due time
|
136
197
|
if ts > now # Future job
|
137
198
|
redis.zadd(rk_deferred, ts, qnid)
|
138
199
|
else
|
139
|
-
|
200
|
+
if push_to_top
|
201
|
+
redis.rpush(rk_queue(queue), qnid)
|
202
|
+
else
|
203
|
+
redis.lpush(rk_queue(queue), qnid)
|
204
|
+
end
|
140
205
|
end
|
141
206
|
end
|
142
207
|
|
@@ -151,103 +216,72 @@ module Rescheduler
|
|
151
216
|
nil
|
152
217
|
end
|
153
218
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
# Make a job immediate if it is not already. Erase the wait
|
170
|
-
def make_immediate(options)
|
171
|
-
dtn = rk_deferred # Make a copy in case prefix changes
|
172
|
-
qnid = get_qnid(options)
|
173
|
-
ntry = 0
|
174
|
-
loop do
|
175
|
-
redis.watch(dtn) do
|
176
|
-
if redis.zcard(dtn, qnid) == 0
|
177
|
-
redis.unwatch(dtn)
|
178
|
-
return # Not a deferred job
|
179
|
-
else
|
180
|
-
redis.multi
|
181
|
-
redis.zrem(dtn, qnid)
|
182
|
-
q = qnid_to_queue(qnid)
|
183
|
-
redis.lpush(rk_queue(q), qnid)
|
184
|
-
if !redis.exec
|
185
|
-
# Contention happens, retrying
|
186
|
-
log_debug("make_immediate contention for #{qnid}")
|
187
|
-
Kernel.sleep (rand(ntry * 1000) / 1000.0) if ntry > 0
|
188
|
-
else
|
189
|
-
return # Done
|
190
|
-
end
|
191
|
-
end
|
219
|
+
# Temp function for special purpose. Completely by-pass concurrency check to increase speed
|
220
|
+
def quick_enqueue_batch(queue, ids, reset = false)
|
221
|
+
argsmap = {}
|
222
|
+
vals = []
|
223
|
+
ids.each do |id|
|
224
|
+
qnid = get_qnid(queue, id)
|
225
|
+
vals << qnid
|
226
|
+
argsmap[rk_args(qnid)] = '{}' # Empty args
|
227
|
+
end unless ids.blank?
|
228
|
+
redis.pipelined do # Should do redis.multi if concurrency is a problem
|
229
|
+
redis.del(rk_queue(queue)) if reset # Empty the list fast
|
230
|
+
unless ids.blank?
|
231
|
+
redis.lpush(rk_queue(queue), vals)
|
232
|
+
argsmap.each { |k,v| redis.set(k,v) }
|
192
233
|
end
|
193
|
-
ntry += 1
|
194
234
|
end
|
235
|
+
nil
|
195
236
|
end
|
196
237
|
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
# TODO
|
204
|
-
end
|
205
|
-
|
206
|
-
# Load state from a file. Will merge into existing jobs if there are any (make sure it is done only once)
|
207
|
-
# This can be done before any worker starts, or after.
|
208
|
-
# Workers still need to be manually started
|
209
|
-
def deserialize(filename)
|
210
|
-
# TODO
|
211
|
-
end
|
212
|
-
|
213
|
-
# Clear redis states and delete all jobs (useful before deserialize)
|
214
|
-
def erase_all
|
215
|
-
# TODO
|
238
|
+
# Returns true if enqueued a new job, otherwise returns false
|
239
|
+
def enqueue_unless_exists(options)
|
240
|
+
# NOTE: There is no point synchronizing exists and enqueue
|
241
|
+
return false if exists?(options)
|
242
|
+
enqueue(options)
|
243
|
+
return true
|
216
244
|
end
|
217
245
|
|
218
|
-
|
246
|
+
#====================================================================
|
219
247
|
# Job definition
|
248
|
+
#====================================================================
|
249
|
+
|
220
250
|
# Task consumer routines
|
221
251
|
def job(tube, &block)
|
222
|
-
|
223
|
-
|
252
|
+
@@runners ||= {}
|
253
|
+
@@runners[tube] = block
|
224
254
|
return nil
|
225
255
|
end
|
226
256
|
|
227
|
-
|
257
|
+
#====================================================================
|
228
258
|
# Error handling
|
259
|
+
#====================================================================
|
229
260
|
def on_error(tube=nil, &block)
|
230
261
|
if tube != nil
|
231
|
-
|
232
|
-
|
262
|
+
@@error_handlers ||= {}
|
263
|
+
@@error_handlers[tube] = block
|
233
264
|
else
|
234
|
-
|
265
|
+
@@global_error_handler = block;
|
235
266
|
end
|
236
267
|
end
|
237
|
-
|
268
|
+
|
269
|
+
#====================================================================
|
238
270
|
# Runner/Maintenance routines
|
271
|
+
#====================================================================
|
239
272
|
def start(*tubes)
|
273
|
+
@@runners ||= {}
|
240
274
|
# Check arguments
|
241
|
-
if
|
275
|
+
if @@runners.size == 0
|
242
276
|
raise Exception, 'Can not start worker without defining job handlers.'
|
243
277
|
end
|
244
278
|
|
245
279
|
tubes.each do |t|
|
246
|
-
next if
|
280
|
+
next if @@runners.include?(t)
|
247
281
|
raise Exception, "Handler for queue #{t} is undefined."
|
248
282
|
end
|
249
283
|
|
250
|
-
tubes =
|
284
|
+
tubes = @@runners.keys if !tubes || tubes.size == 0
|
251
285
|
|
252
286
|
log_debug "[[ Starting: #{tubes.join(',')} ]]"
|
253
287
|
|
@@ -257,20 +291,33 @@ module Rescheduler
|
|
257
291
|
keys = tubes.map {|t| rk_queue(t)}
|
258
292
|
keys << rk_maintenace
|
259
293
|
|
294
|
+
# Queue to control a named worker
|
295
|
+
worker_queue = Worker.rk_queue if Worker.named?
|
296
|
+
keys.unshift(worker_queue) if worker_queue # worker control queue is the first we respond to
|
297
|
+
|
260
298
|
dopush = nil
|
261
299
|
|
262
|
-
|
300
|
+
@@end_job_loop = false
|
301
|
+
while !@@end_job_loop
|
263
302
|
# Run maintenance and determine timeout
|
264
303
|
next_job_time = determine_next_deferred_job_time.to_i
|
265
304
|
|
266
305
|
if dopush # Only pass-on the token after we are done with maintenance. Avoid contention
|
267
|
-
redis.lpush(rk_maintenace, dopush)
|
306
|
+
redis.lpush(rk_maintenace, dopush)
|
268
307
|
dopush = nil
|
269
308
|
end
|
270
309
|
|
271
310
|
# Blocking wait
|
272
311
|
timeout = next_job_time - Time.now.to_i
|
273
312
|
timeout = 1 if timeout < 1
|
313
|
+
|
314
|
+
# A producer may insert another job after BRPOP and before WATCH
|
315
|
+
# Due to limitations of BRPOP we can not prevent this from happening.
|
316
|
+
# When it happens we will consume the args of the later job, causing
|
317
|
+
# the newly inserted job to be "promoted" to the front of the queue
|
318
|
+
# This may not be desirable...
|
319
|
+
# (too bad BRPOPLPUSH does not support multiple queues...)
|
320
|
+
# TODO: Maybe LUA script is the way out of this.
|
274
321
|
result = redis.brpop(keys, :timeout=>timeout)
|
275
322
|
|
276
323
|
# Handle task
|
@@ -278,25 +325,39 @@ module Rescheduler
|
|
278
325
|
tube = result[0]
|
279
326
|
qnid = result[1]
|
280
327
|
if tube == rk_maintenace
|
281
|
-
# Circulate maintenance task until it comes a full circle. This depends on redis
|
282
|
-
# first come first serve policy in brpop.
|
328
|
+
# Circulate maintenance task until it comes a full circle. This depends on redis
|
329
|
+
# first come first serve policy in brpop.
|
283
330
|
dopush = qnid + client_key unless qnid.include?(client_key) # Push if we have not pushed yet.
|
331
|
+
elsif tube == worker_queue
|
332
|
+
Worker.handle_command(qnid)
|
284
333
|
else
|
285
334
|
run_job(qnid)
|
286
335
|
end
|
287
|
-
else
|
336
|
+
else
|
288
337
|
# Do nothing when got timeout, the run_maintenance will take care of deferred jobs
|
289
338
|
end
|
290
339
|
end
|
291
340
|
end
|
292
341
|
|
293
|
-
|
342
|
+
def end_job_loop; @@end_job_loop = true; end
|
343
|
+
|
344
|
+
|
345
|
+
# Logging facility
|
346
|
+
def log_debug(msg)
|
347
|
+
return if config[:silent]
|
348
|
+
print("#{Time.now.iso8601} #{msg}\n")
|
349
|
+
STDOUT.flush
|
350
|
+
end
|
351
|
+
|
352
|
+
#====================================================================
|
353
|
+
private
|
354
|
+
#====================================================================
|
294
355
|
|
295
356
|
# Internal routines operating out of qnid
|
296
357
|
def idelete(qnid)
|
297
358
|
queue = qnid.split(':').first
|
298
359
|
redis.multi do
|
299
|
-
redis.
|
360
|
+
redis.del(rk_args(qnid))
|
300
361
|
redis.zrem(rk_deferred, qnid)
|
301
362
|
redis.lrem(rk_queue(queue), 0, qnid)
|
302
363
|
end
|
@@ -306,74 +367,70 @@ module Rescheduler
|
|
306
367
|
def run_job(qnid)
|
307
368
|
# 1. load job parameters for running
|
308
369
|
optstr = nil
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
redis.
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
redis.hset(rk_running, qnid, optstr)
|
322
|
-
end
|
323
|
-
if !res
|
324
|
-
# Contention, try read again
|
325
|
-
log_debug("Job read contention: (#{qnid})")
|
326
|
-
end
|
327
|
-
end
|
328
|
-
end until res
|
370
|
+
key = rk_args(qnid)
|
371
|
+
# Atomic get and delete the arg
|
372
|
+
redis.multi do
|
373
|
+
optstr = redis.get(key)
|
374
|
+
redis.del(key)
|
375
|
+
end
|
376
|
+
|
377
|
+
optstr = optstr.value # get the value from the multi block future
|
378
|
+
if optstr.nil?
|
379
|
+
log_debug("Job is deleted mysteriously: (#{qnid})")
|
380
|
+
return # Job is deleted somewhere
|
381
|
+
end
|
329
382
|
|
330
383
|
# Parse and run
|
331
384
|
sopt = MultiJson.load(optstr, :symbolize_keys => true)
|
385
|
+
queue,id = qnid.split(':', 2)
|
386
|
+
sopt[:queue] ||= queue
|
387
|
+
sopt[:id] ||= id
|
332
388
|
|
333
389
|
# Handle parameters
|
334
390
|
if (sopt.include?(:recur_every))
|
335
391
|
newopt = sopt.dup
|
336
392
|
newopt[:due_at] = (sopt[:due_at] || Time.now).to_i + sopt[:recur_every].to_i
|
337
393
|
newopt.delete(:due_in) # In case the first job was specified by :due_in
|
338
|
-
log_debug("---Enqueue #{qnid}:
|
394
|
+
log_debug("---Enqueue #{qnid}: recur_every #{sopt[:recur_every]}")
|
339
395
|
enqueue(newopt)
|
340
396
|
end
|
341
397
|
|
342
|
-
if (sopt.include?(:recur_daily))
|
343
|
-
newopt = sopt.dup
|
344
|
-
newopt
|
345
|
-
newopt.delete(:due_in) #
|
346
|
-
log_debug("---Enqueue #{qnid}:
|
398
|
+
if (sopt.include?(:recur_daily) || sopt.include?(:recur_weekly))
|
399
|
+
newopt = sopt.dup
|
400
|
+
newopt.delete(:due_at)
|
401
|
+
newopt.delete(:due_in) # No more due info, just the recurrance
|
402
|
+
log_debug("---Enqueue #{qnid}: recur_daily #{sopt[:recur_daily]}") if sopt.include?(:recur_daily)
|
403
|
+
log_debug("---Enqueue #{qnid}: recur_weekly #{sopt[:recur_weekly]}") if sopt.include?(:recur_weekly)
|
347
404
|
enqueue(newopt)
|
348
405
|
end
|
349
406
|
|
350
407
|
# 2. Find runner and invoke it
|
351
408
|
begin
|
352
409
|
log_debug(">>---- Starting #{qnid}")
|
353
|
-
runner =
|
410
|
+
runner = @@runners[qnid_to_queue(qnid)]
|
354
411
|
if runner.is_a?(Proc)
|
355
412
|
runner.call(sopt)
|
356
413
|
log_debug("----<< Finished #{qnid}")
|
414
|
+
Worker.inc_job_count # Stats for the worker
|
357
415
|
else
|
358
416
|
log_debug("----<< Failed #{qnid}: Unknown queue name, handler not defined")
|
359
417
|
end
|
360
418
|
rescue Exception => e
|
361
419
|
log_debug("----<< Failed #{qnid}: -------------\n #{$!}")
|
362
420
|
log_debug(e.backtrace[0..4].join("\n"))
|
363
|
-
handle_error(e,
|
421
|
+
handle_error(e, queue, sopt)
|
364
422
|
log_debug("------------------------------------\n")
|
365
423
|
end
|
366
|
-
|
367
|
-
# 3. Remove job from running list (Done)
|
368
|
-
redis.hdel(rk_running, qnid)
|
369
424
|
end
|
370
425
|
|
371
|
-
def handle_error(e,
|
372
|
-
|
426
|
+
def handle_error(e, queue, sopt)
|
427
|
+
@@error_handlers ||= {}
|
428
|
+
@@global_error_handler ||= nil
|
429
|
+
error_handler = @@error_handlers[queue]
|
373
430
|
if error_handler
|
374
|
-
error_handler.call(e, sopt)
|
375
|
-
elsif
|
376
|
-
|
431
|
+
error_handler.call(e, sopt)
|
432
|
+
elsif @@global_error_handler
|
433
|
+
@@global_error_handler.call(e, sopt)
|
377
434
|
end
|
378
435
|
end
|
379
436
|
|
@@ -383,17 +440,18 @@ module Rescheduler
|
|
383
440
|
def service_deferred_jobs
|
384
441
|
dtn = rk_deferred # Make a copy in case prefix changes
|
385
442
|
ntry = 0
|
386
|
-
|
443
|
+
while ntry < 6 do
|
387
444
|
curtime = Time.now.to_i
|
445
|
+
return if redis.zcount(dtn, 0, curtime) == 0
|
446
|
+
|
447
|
+
limit = ntry < 3 ? 100 : 1 # After first 3 tries, do just 1
|
388
448
|
redis.watch(dtn) do
|
389
|
-
tasks = redis.zrangebyscore(dtn, 0, curtime)
|
449
|
+
tasks = redis.zrangebyscore(dtn, 0, curtime, :limit=>[0,limit]) # Serve at most 100
|
390
450
|
if tasks.empty?
|
391
451
|
redis.unwatch
|
392
452
|
return # Nothing to transfer, moving on.
|
393
453
|
end
|
394
454
|
|
395
|
-
redis.multi
|
396
|
-
redis.zremrangebyscore(dtn, 0, curtime)
|
397
455
|
to_push = {}
|
398
456
|
tasks.each do |qnid|
|
399
457
|
q = rk_queue(qnid_to_queue(qnid))
|
@@ -401,6 +459,9 @@ module Rescheduler
|
|
401
459
|
to_push[q] << qnid
|
402
460
|
end
|
403
461
|
|
462
|
+
redis.multi
|
463
|
+
redis.zrem(dtn, tasks)
|
464
|
+
|
404
465
|
to_push.each do |q, qnids|
|
405
466
|
redis.lpush(q, qnids) # Batch command
|
406
467
|
end
|
@@ -409,63 +470,30 @@ module Rescheduler
|
|
409
470
|
# Contention happens, retrying
|
410
471
|
# Sleep a random amount of time after first try
|
411
472
|
ntry += 1
|
412
|
-
log_debug("service_deferred_jobs contention")
|
473
|
+
log_debug("service_deferred_jobs(#{limit}) contention")
|
413
474
|
Kernel.sleep (rand(ntry * 1000) / 1000.0)
|
414
475
|
else
|
415
476
|
return # Done transfering
|
416
477
|
end
|
417
478
|
end
|
418
|
-
|
419
|
-
if ntry > 3 # Max number of tries
|
420
|
-
# Fall back to
|
421
|
-
service_one_deferred_job
|
422
|
-
return
|
423
|
-
end
|
424
479
|
end
|
425
|
-
end
|
426
|
-
|
427
|
-
def service_one_deferred_jobs
|
428
|
-
dtn = rk_deferred # Make a copy in case prefix changes
|
429
|
-
ntry = 0
|
430
|
-
curtime = Time.now.to_i
|
431
|
-
loop do
|
432
|
-
redis.watch(dtn) do
|
433
|
-
tasks = redis.zrangebyscore(dtn, 0, curtime, :limit=>[0,1])
|
434
|
-
if tasks.empty?
|
435
|
-
redis.unwatch
|
436
|
-
return # Nothing to transfer, moving on.
|
437
|
-
end
|
438
480
|
|
439
|
-
|
440
|
-
q = qnid_to_queue(qnid)
|
441
|
-
|
442
|
-
redis.multi
|
443
|
-
redis.zrem(dtn, qnid)
|
444
|
-
redis.lpush(rk_queue(q), qnid)
|
445
|
-
if !redis.exec
|
446
|
-
# Contention happens, retrying
|
447
|
-
# Sleep a random amount of time after first try
|
448
|
-
log_debug("service_one_deferred_job contention")
|
449
|
-
ntry += 1
|
450
|
-
Kernel.sleep (rand(ntry * 1000) / 1000.0)
|
451
|
-
else
|
452
|
-
break # Done transfering one job
|
453
|
-
end
|
454
|
-
end
|
455
|
-
end
|
481
|
+
log_debug("service_deferred_jobs failed, will try next time")
|
456
482
|
end
|
457
483
|
|
458
484
|
def determine_next_deferred_job_time(skip_service = nil)
|
459
485
|
tsnow = Time.now.to_f
|
460
|
-
maxtime = tsnow + 3600
|
486
|
+
maxtime = tsnow + 3600 + rand(100) # Randomize wake time to avoid multi worker service contention
|
461
487
|
|
462
488
|
dt = redis.zrange(rk_deferred, 0, 0, :with_scores=>true)[0]
|
463
489
|
nt = (dt && dt[1] && dt[1] < maxtime) ? dt[1] : maxtime
|
464
490
|
if !skip_service && nt <= tsnow
|
465
|
-
|
466
|
-
|
467
|
-
|
468
|
-
|
491
|
+
do_if_can_acquire_semaphore do
|
492
|
+
service_deferred_jobs
|
493
|
+
# Get the deferred jobs again.
|
494
|
+
dt = redis.zrange(rk_deferred, 0, 0, :with_scores=>true)[0]
|
495
|
+
nt = (dt && dt[1] && dt[1] < maxtime) ? dt[1] : maxtime
|
496
|
+
end
|
469
497
|
end
|
470
498
|
return nt
|
471
499
|
end
|
@@ -474,50 +502,99 @@ module Rescheduler
|
|
474
502
|
|
475
503
|
def rk_deferred; prefix + 'TMDEFERRED'; end
|
476
504
|
def rk_maintenace; prefix + 'TMMAINT'; end
|
477
|
-
def rk_args; prefix
|
478
|
-
def rk_running; prefix + "TMRUNNING"; end
|
505
|
+
def rk_args(qnid); "#{prefix}TMARGS:#{qnid}"; end
|
479
506
|
def rk_counter; prefix + 'TMCOUNTER'; end
|
507
|
+
def rk_worker_semaphore; prefix + 'TMWORKERLOCK'; end # This is a boolean with a timeout for workers to exclude each other
|
508
|
+
|
509
|
+
# None blocking, returns true if semaphore is acquired (for a given timeout), this is cooperative to avoid guaranteed contentions
|
510
|
+
def try_acquire_semaphore(timeout=300) # Default for 5 minutes, there must be a timeout
|
511
|
+
semkey = rk_worker_semaphore
|
512
|
+
if redis.setnx(semkey, 1) # Any value would be fine
|
513
|
+
redis.expire(semkey, timeout)
|
514
|
+
return true
|
515
|
+
else
|
516
|
+
# Already created, someone has it
|
517
|
+
return false
|
518
|
+
end
|
519
|
+
end
|
520
|
+
|
521
|
+
# This releases semaphore unconditionally
|
522
|
+
def release_semaphore
|
523
|
+
# NOTE: There is a chance we remove the lock created by another worker after our own expired.
|
524
|
+
# This is OK since the lock is cooperative and not necessary (real locking is done through contention checks)
|
525
|
+
redis.del(rk_worker_semaphore)
|
526
|
+
end
|
480
527
|
|
481
|
-
|
482
|
-
|
528
|
+
# Run block only if the semaphore can be acquired, otherwise do nothing
|
529
|
+
def do_if_can_acquire_semaphore
|
530
|
+
if try_acquire_semaphore
|
531
|
+
yield
|
532
|
+
release_semaphore
|
533
|
+
end
|
483
534
|
end
|
484
535
|
|
536
|
+
def get_qnid(queue, id); return "#{queue}:#{id}"; end
|
537
|
+
|
485
538
|
def qnid_to_queue(qnid)
|
486
539
|
idx = qnid.index(':')
|
487
540
|
unless idx
|
488
541
|
log_debug("Invalid qnid: #{qnid}")
|
489
|
-
return nil
|
542
|
+
return nil
|
490
543
|
end
|
491
544
|
qnid[0...idx]
|
492
545
|
end
|
493
546
|
|
494
547
|
def redis
|
495
|
-
|
548
|
+
@@redis ||= config[:redis] || Redis.new(config[:redis_connection] || {})
|
496
549
|
end
|
497
550
|
|
498
551
|
def validate_queue_name(queue)
|
499
552
|
raise ArgumentError, 'Queue name can not contain special characters' if queue.include?(':')
|
500
553
|
end
|
501
554
|
|
502
|
-
def parse_seconds_of_day(recur_daily)
|
503
|
-
return recur_daily if recur_daily.is_a?(Fixnum)
|
504
|
-
time = Time.parse(recur_daily)
|
505
|
-
return time.to_i - Time.new(time.year, time.month, time.day).to_i
|
506
|
-
end
|
507
|
-
|
508
555
|
# Find the next recur time
|
509
556
|
def time_from_recur_daily(recur_daily, now = Time.now)
|
510
|
-
|
511
|
-
|
512
|
-
|
513
|
-
|
557
|
+
parsed = Date._parse(recur_daily)
|
558
|
+
if !parsed[:hour] || (parsed.keys - [:zone, :hour, :min, :sec, :offset, :sec_fraction]).present?
|
559
|
+
raise ArgumentError, 'Unexpected recur_daily value: ' + recur_daily
|
560
|
+
end
|
561
|
+
|
562
|
+
if !parsed[:offset]
|
563
|
+
raise ArgumentError, 'A timezone is required for recur_daily: ' + recur_daily
|
564
|
+
end
|
565
|
+
|
566
|
+
# Never offset over one day (e.g. 23:59 PDT)
|
567
|
+
offset = (parsed[:hour] * 3600 + (parsed[:min]||0) * 60 + (parsed[:sec] || 0) - parsed[:offset]) % 86400
|
568
|
+
|
569
|
+
t = Time.utc(now.year, now.month, now.day) + offset
|
570
|
+
t += 86400 if t <= now + 1
|
571
|
+
return t
|
572
|
+
end
|
573
|
+
|
574
|
+
def time_from_recur_weekly(recur_weekly, now = Time.now)
|
575
|
+
parsed = Date._parse(recur_weekly)
|
576
|
+
if !parsed[:hour] || !parsed[:wday] || (parsed.keys - [:wday, :zone, :hour, :min, :sec, :offset, :sec_fraction]).present?
|
577
|
+
raise ArgumentError, 'Unexpected recur_weekly value: ' + recur_weekly
|
578
|
+
end
|
579
|
+
|
580
|
+
if !parsed[:offset]
|
581
|
+
raise ArgumentError, 'A timezone is required for recur_weekly: ' + recur_weekly
|
582
|
+
end
|
583
|
+
|
584
|
+
# Never offset over one week
|
585
|
+
offset = parsed[:hour] * 3600 + (parsed[:min]||0) * 60 + (parsed[:sec] || 0) - parsed[:offset]
|
586
|
+
offset = (offset + parsed[:wday] * 86400) % (86400 * 7)
|
587
|
+
|
588
|
+
t = Time.utc(now.year, now.month, now.day) - now.wday * 86400 + offset
|
589
|
+
t += 86400 * 7 if t <= now + 1
|
590
|
+
return t
|
514
591
|
end
|
515
592
|
|
516
593
|
def validate_recurrance(options)
|
517
594
|
rcnt = 0
|
518
595
|
if (options.include?(:recur_every))
|
519
596
|
rcnt += 1
|
520
|
-
raise 'Expect integer for :recur_every parameter' unless options[:recur_every].is_a?(
|
597
|
+
raise 'Expect integer for :recur_every parameter' unless options[:recur_every].is_a?(Integer)
|
521
598
|
end
|
522
599
|
|
523
600
|
if (options.include?(:recur_daily))
|
@@ -527,14 +604,16 @@ module Rescheduler
|
|
527
604
|
options[:due_at] = time # Setup the first run
|
528
605
|
end
|
529
606
|
end
|
530
|
-
raise 'Can only specify one recurrance parameter' if rcnt > 1
|
531
|
-
end
|
532
607
|
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
608
|
+
if (options.include?(:recur_weekly))
|
609
|
+
rcnt += 1
|
610
|
+
time = time_from_recur_weekly(options[:recur_weekly]) # Try parse and make sure we can
|
611
|
+
unless options.include?(:due_at) || options.include?(:due_in)
|
612
|
+
options[:due_at] = time # Setup the first run
|
613
|
+
end
|
614
|
+
end
|
615
|
+
|
616
|
+
raise 'Can only specify one recurrance parameter' if rcnt > 1
|
538
617
|
end
|
539
618
|
|
540
619
|
end
|
metadata
CHANGED
@@ -1,80 +1,77 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rescheduler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.5.1
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Dongyi Liao
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2016-06-04 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: redis
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: multi_json
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
description: Rescheduler is a library that uses Redis to maintain a task queue of
|
47
42
|
immediate and delayed jobs without polling.
|
48
43
|
email: liaody@gmail.com
|
49
|
-
executables:
|
44
|
+
executables:
|
45
|
+
- rescheduler_launch
|
50
46
|
extensions: []
|
51
47
|
extra_rdoc_files: []
|
52
48
|
files:
|
49
|
+
- bin/rescheduler_launch
|
53
50
|
- lib/rescheduler.rb
|
51
|
+
- lib/rescheduler/sync.rb
|
52
|
+
- lib/rescheduler/worker.rb
|
54
53
|
homepage: https://github.com/liaody/rescheduler
|
55
54
|
licenses:
|
56
55
|
- BSD
|
56
|
+
metadata: {}
|
57
57
|
post_install_message:
|
58
58
|
rdoc_options: []
|
59
59
|
require_paths:
|
60
60
|
- lib
|
61
61
|
required_ruby_version: !ruby/object:Gem::Requirement
|
62
|
-
none: false
|
63
62
|
requirements:
|
64
|
-
- -
|
63
|
+
- - ">="
|
65
64
|
- !ruby/object:Gem::Version
|
66
65
|
version: '0'
|
67
66
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
68
|
-
none: false
|
69
67
|
requirements:
|
70
|
-
- -
|
68
|
+
- - ">="
|
71
69
|
- !ruby/object:Gem::Version
|
72
70
|
version: '0'
|
73
71
|
requirements: []
|
74
72
|
rubyforge_project:
|
75
|
-
rubygems_version:
|
73
|
+
rubygems_version: 2.5.1
|
76
74
|
signing_key:
|
77
|
-
specification_version:
|
75
|
+
specification_version: 4
|
78
76
|
summary: A job queue for immediate and delayed jobs using Redis
|
79
77
|
test_files: []
|
80
|
-
has_rdoc:
|