qless 0.9.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +9 -3
- data/README.md +70 -25
- data/Rakefile +125 -9
- data/exe/install_phantomjs +21 -0
- data/lib/qless.rb +115 -76
- data/lib/qless/config.rb +11 -9
- data/lib/qless/failure_formatter.rb +43 -0
- data/lib/qless/job.rb +201 -102
- data/lib/qless/job_reservers/ordered.rb +7 -1
- data/lib/qless/job_reservers/round_robin.rb +16 -6
- data/lib/qless/job_reservers/shuffled_round_robin.rb +9 -2
- data/lib/qless/lua/qless-lib.lua +2463 -0
- data/lib/qless/lua/qless.lua +2012 -0
- data/lib/qless/lua_script.rb +63 -12
- data/lib/qless/middleware/memory_usage_monitor.rb +62 -0
- data/lib/qless/middleware/metriks.rb +45 -0
- data/lib/qless/middleware/redis_reconnect.rb +6 -3
- data/lib/qless/middleware/requeue_exceptions.rb +94 -0
- data/lib/qless/middleware/retry_exceptions.rb +38 -9
- data/lib/qless/middleware/sentry.rb +3 -7
- data/lib/qless/middleware/timeout.rb +64 -0
- data/lib/qless/queue.rb +90 -55
- data/lib/qless/server.rb +177 -130
- data/lib/qless/server/views/_job.erb +33 -15
- data/lib/qless/server/views/completed.erb +11 -0
- data/lib/qless/server/views/layout.erb +70 -11
- data/lib/qless/server/views/overview.erb +93 -53
- data/lib/qless/server/views/queue.erb +9 -8
- data/lib/qless/server/views/queues.erb +18 -1
- data/lib/qless/subscriber.rb +37 -22
- data/lib/qless/tasks.rb +5 -10
- data/lib/qless/test_helpers/worker_helpers.rb +55 -0
- data/lib/qless/version.rb +3 -1
- data/lib/qless/worker.rb +4 -413
- data/lib/qless/worker/base.rb +247 -0
- data/lib/qless/worker/forking.rb +245 -0
- data/lib/qless/worker/serial.rb +41 -0
- metadata +135 -52
- data/lib/qless/qless-core/cancel.lua +0 -101
- data/lib/qless/qless-core/complete.lua +0 -233
- data/lib/qless/qless-core/config.lua +0 -56
- data/lib/qless/qless-core/depends.lua +0 -65
- data/lib/qless/qless-core/deregister_workers.lua +0 -12
- data/lib/qless/qless-core/fail.lua +0 -117
- data/lib/qless/qless-core/failed.lua +0 -83
- data/lib/qless/qless-core/get.lua +0 -37
- data/lib/qless/qless-core/heartbeat.lua +0 -51
- data/lib/qless/qless-core/jobs.lua +0 -41
- data/lib/qless/qless-core/pause.lua +0 -18
- data/lib/qless/qless-core/peek.lua +0 -165
- data/lib/qless/qless-core/pop.lua +0 -314
- data/lib/qless/qless-core/priority.lua +0 -32
- data/lib/qless/qless-core/put.lua +0 -169
- data/lib/qless/qless-core/qless-lib.lua +0 -2354
- data/lib/qless/qless-core/qless.lua +0 -1862
- data/lib/qless/qless-core/queues.lua +0 -58
- data/lib/qless/qless-core/recur.lua +0 -190
- data/lib/qless/qless-core/retry.lua +0 -73
- data/lib/qless/qless-core/stats.lua +0 -92
- data/lib/qless/qless-core/tag.lua +0 -100
- data/lib/qless/qless-core/track.lua +0 -79
- data/lib/qless/qless-core/unfail.lua +0 -54
- data/lib/qless/qless-core/unpause.lua +0 -12
- data/lib/qless/qless-core/workers.lua +0 -69
- data/lib/qless/wait_until.rb +0 -19
data/lib/qless/lua_script.rb
CHANGED
@@ -1,8 +1,13 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
1
3
|
require 'digest/sha1'
|
2
4
|
|
3
5
|
module Qless
|
6
|
+
LuaScriptError = Class.new(Qless::Error)
|
7
|
+
|
8
|
+
# Wraps a lua script. Knows how to reload it if necessary
|
4
9
|
class LuaScript
|
5
|
-
|
10
|
+
SCRIPT_ROOT = File.expand_path('../lua', __FILE__)
|
6
11
|
|
7
12
|
def initialize(name, redis)
|
8
13
|
@name = name
|
@@ -12,31 +17,77 @@ module Qless
|
|
12
17
|
|
13
18
|
attr_reader :name, :redis, :sha
|
14
19
|
|
15
|
-
def reload
|
20
|
+
def reload
|
16
21
|
@sha = @redis.script(:load, script_contents)
|
17
22
|
end
|
18
23
|
|
19
|
-
def call(
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
+
def call(*argv)
|
25
|
+
handle_no_script_error do
|
26
|
+
_call(*argv)
|
27
|
+
end
|
28
|
+
rescue Redis::CommandError => err
|
29
|
+
if match = err.message.match('user_script:\d+:\s*(\w+.+$)')
|
30
|
+
raise LuaScriptError.new(match[1])
|
31
|
+
else
|
32
|
+
raise err
|
33
|
+
end
|
24
34
|
end
|
25
35
|
|
26
36
|
private
|
27
37
|
|
28
38
|
if USING_LEGACY_REDIS_VERSION
|
29
|
-
def _call(
|
30
|
-
@redis.evalsha(@sha,
|
39
|
+
def _call(*argv)
|
40
|
+
@redis.evalsha(@sha, 0, *argv)
|
31
41
|
end
|
32
42
|
else
|
33
|
-
def _call(
|
34
|
-
@redis.evalsha(@sha, keys:
|
43
|
+
def _call(*argv)
|
44
|
+
@redis.evalsha(@sha, keys: [], argv: argv)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_no_script_error
|
49
|
+
yield
|
50
|
+
rescue ScriptNotLoadedRedisCommandError
|
51
|
+
reload
|
52
|
+
yield
|
53
|
+
end
|
54
|
+
|
55
|
+
# Module for notifying when a script hasn't yet been loaded
|
56
|
+
module ScriptNotLoadedRedisCommandError
|
57
|
+
MESSAGE = 'NOSCRIPT No matching script. Please use EVAL.'
|
58
|
+
|
59
|
+
def self.===(error)
|
60
|
+
error.is_a?(Redis::CommandError) && error.message == MESSAGE
|
35
61
|
end
|
36
62
|
end
|
37
63
|
|
38
64
|
def script_contents
|
39
|
-
@script_contents ||= File.read(File.join(
|
65
|
+
@script_contents ||= File.read(File.join(SCRIPT_ROOT, "#{@name}.lua"))
|
40
66
|
end
|
41
67
|
end
|
68
|
+
|
69
|
+
# Provides a simple way to load and use lua-based Qless plugins.
|
70
|
+
# This combines the qless-lib.lua script plus your custom script
|
71
|
+
# contents all into one script, so that your script can use
|
72
|
+
# Qless's lua API.
|
73
|
+
class LuaPlugin < LuaScript
|
74
|
+
def initialize(name, redis, plugin_contents)
|
75
|
+
@name = name
|
76
|
+
@redis = redis
|
77
|
+
@plugin_contents = plugin_contents.gsub(COMMENT_LINES_RE, '')
|
78
|
+
super(name, redis)
|
79
|
+
end
|
80
|
+
|
81
|
+
private
|
82
|
+
|
83
|
+
def script_contents
|
84
|
+
@script_contents ||= [QLESS_LIB_CONTENTS, @plugin_contents].join("\n\n")
|
85
|
+
end
|
86
|
+
|
87
|
+
COMMENT_LINES_RE = /^\s*--.*$\n?/
|
88
|
+
|
89
|
+
QLESS_LIB_CONTENTS = File.read(
|
90
|
+
File.join(SCRIPT_ROOT, 'qless-lib.lua')
|
91
|
+
).gsub(COMMENT_LINES_RE, '')
|
92
|
+
end
|
42
93
|
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
|
2
|
+
module Qless
|
3
|
+
module Middleware
|
4
|
+
# Monitors the memory usage of the Qless worker, instructing
|
5
|
+
# it to shutdown when memory exceeds the given :max_memory threshold.
|
6
|
+
class MemoryUsageMonitor < Module
|
7
|
+
def initialize(options)
|
8
|
+
max_memory = options.fetch(:max_memory)
|
9
|
+
|
10
|
+
module_eval do
|
11
|
+
job_counter = 0
|
12
|
+
|
13
|
+
define_method :around_perform do |job|
|
14
|
+
job_counter += 1
|
15
|
+
|
16
|
+
begin
|
17
|
+
super(job)
|
18
|
+
ensure
|
19
|
+
current_mem = MemoryUsageMonitor.current_usage_in_kb
|
20
|
+
if current_mem > max_memory
|
21
|
+
log(:info, "Exiting after job #{job_counter} since current memory " \
|
22
|
+
"(#{current_mem} KB) has exceeded max allowed memory " \
|
23
|
+
"(#{max_memory} KB).")
|
24
|
+
shutdown
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
SHELL_OUT_FOR_MEMORY = -> do
|
32
|
+
# Taken from:
|
33
|
+
# http://stackoverflow.com/a/4133642/29262
|
34
|
+
Integer(`ps -o rss= -p #{Process.pid}`)
|
35
|
+
end unless defined?(SHELL_OUT_FOR_MEMORY)
|
36
|
+
|
37
|
+
begin
|
38
|
+
require 'rusage'
|
39
|
+
rescue LoadError
|
40
|
+
warn "Could not load `rusage` gem. Falling back to shelling out "
|
41
|
+
"to get process memory usage, which is several orders of magnitude slower."
|
42
|
+
|
43
|
+
define_singleton_method(:current_usage_in_kb, &SHELL_OUT_FOR_MEMORY)
|
44
|
+
else
|
45
|
+
memory_ratio = Process.rusage.maxrss / SHELL_OUT_FOR_MEMORY.().to_f
|
46
|
+
|
47
|
+
if (800...1200).cover?(memory_ratio)
|
48
|
+
# OS X tends to return maxrss in Bytes.
|
49
|
+
def self.current_usage_in_kb
|
50
|
+
Process.rusage.maxrss / 1024
|
51
|
+
end
|
52
|
+
else
|
53
|
+
# Linux tends to return maxrss in KB.
|
54
|
+
def self.current_usage_in_kb
|
55
|
+
Process.rusage.maxrss
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'metriks'
|
4
|
+
|
5
|
+
module Qless
|
6
|
+
module Middleware
|
7
|
+
module Metriks
|
8
|
+
|
9
|
+
# Tracks the time jobs take, grouping the timings by the job class.
|
10
|
+
module TimeJobsByClass
|
11
|
+
def around_perform(job)
|
12
|
+
::Metriks.timer("qless.job-times.#{job.klass_name}").time do
|
13
|
+
super
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
# Increments a counter each time an instance of a particular job class
|
19
|
+
# completes.
|
20
|
+
#
|
21
|
+
# Usage:
|
22
|
+
#
|
23
|
+
# Qless::Worker.class_eval do
|
24
|
+
# include Qless::Middleware::CountEvents.new(
|
25
|
+
# SomeJobClass => "event_name",
|
26
|
+
# SomeOtherJobClass => "some_other_event"
|
27
|
+
# )
|
28
|
+
# end
|
29
|
+
class CountEvents < Module
|
30
|
+
def initialize(class_to_event_map)
|
31
|
+
module_eval do # eval the block within the module instance
|
32
|
+
define_method :around_perform do |job|
|
33
|
+
super(job)
|
34
|
+
return unless job.state == 'complete'
|
35
|
+
return unless event_name = class_to_event_map[job.klass]
|
36
|
+
|
37
|
+
counter = ::Metriks.counter("qless.job-events.#{event_name}")
|
38
|
+
counter.increment
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -1,13 +1,17 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
1
3
|
module Qless
|
2
4
|
module Middleware
|
5
|
+
# A module for reconnecting to redis for each job
|
3
6
|
module RedisReconnect
|
4
7
|
def self.new(*redis_connections, &block)
|
5
8
|
Module.new do
|
6
9
|
define_singleton_method :to_s do
|
7
|
-
|
10
|
+
'Qless::Middleware::RedisReconnect'
|
8
11
|
end
|
12
|
+
define_singleton_method(:inspect, method(:to_s))
|
9
13
|
|
10
|
-
block ||=
|
14
|
+
block ||= ->(job) { redis_connections }
|
11
15
|
|
12
16
|
define_method :around_perform do |job|
|
13
17
|
Array(block.call(job)).each do |redis|
|
@@ -21,4 +25,3 @@ module Qless
|
|
21
25
|
end
|
22
26
|
end
|
23
27
|
end
|
24
|
-
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
module Qless
|
4
|
+
module Middleware
|
5
|
+
# This middleware is like RetryExceptions, but it doesn't use qless-core's
|
6
|
+
# internal retry/retry-tracking mechanism. Instead, it re-queues the job
|
7
|
+
# when it fails with a matched error, and increments a counter in the job's
|
8
|
+
# data.
|
9
|
+
#
|
10
|
+
# This is useful for exceptions for which you want a different
|
11
|
+
# backoff/retry strategy. The internal retry mechanism doesn't allow for
|
12
|
+
# separate tracking by exception type, and thus doesn't allow you to retry
|
13
|
+
# different exceptions a different number of times.
|
14
|
+
#
|
15
|
+
# This is particularly useful for handling resource throttling errors,
|
16
|
+
# where you may not want exponential backoff, and you may want the error
|
17
|
+
# to be retried many times, w/o having other transient errors retried so
|
18
|
+
# many times.
|
19
|
+
module RequeueExceptions
|
20
|
+
RequeueableException = Struct.new(:klass, :delay_min, :delay_span, :max_attempts) do
|
21
|
+
def self.from_splat_and_options(*klasses, options)
|
22
|
+
delay_range = options.fetch(:delay_range)
|
23
|
+
delay_min = Float(delay_range.min)
|
24
|
+
delay_span = Float(delay_range.max) - Float(delay_range.min)
|
25
|
+
max_attempts = options.fetch(:max_attempts)
|
26
|
+
klasses.map do |klass|
|
27
|
+
new(klass, delay_min, delay_span, max_attempts)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def delay
|
32
|
+
delay_min + Random.rand(delay_span)
|
33
|
+
end
|
34
|
+
|
35
|
+
def raise_if_exhausted_requeues(error, requeues)
|
36
|
+
raise error if requeues >= max_attempts
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def requeue_on(*exceptions, options)
|
41
|
+
RequeueableException.from_splat_and_options(
|
42
|
+
*exceptions, options).each do |exc|
|
43
|
+
requeueable_exceptions[exc.klass] = exc
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
DEFAULT_ON_REQUEUE_CALLBACK = lambda { |error, job| }
|
48
|
+
def use_on_requeue_callback(&block)
|
49
|
+
@on_requeue_callback = block if block
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_requeue_callback
|
53
|
+
@on_requeue_callback ||= DEFAULT_ON_REQUEUE_CALLBACK
|
54
|
+
end
|
55
|
+
|
56
|
+
def handle_exception(job, error)
|
57
|
+
config = requeuable_exception_for(error)
|
58
|
+
|
59
|
+
requeues_by_exception = (job.data['requeues_by_exception'] ||= {})
|
60
|
+
requeues_by_exception[config.klass.name] ||= 0
|
61
|
+
|
62
|
+
config.raise_if_exhausted_requeues(
|
63
|
+
error, requeues_by_exception[config.klass.name])
|
64
|
+
|
65
|
+
requeues_by_exception[config.klass.name] += 1
|
66
|
+
job.requeue(job.queue_name, delay: config.delay, data: job.data)
|
67
|
+
|
68
|
+
on_requeue_callback.call(error, job)
|
69
|
+
end
|
70
|
+
|
71
|
+
def around_perform(job)
|
72
|
+
super
|
73
|
+
rescue *requeueable_exceptions.keys => e
|
74
|
+
handle_exception(job, e)
|
75
|
+
end
|
76
|
+
|
77
|
+
def requeueable?(exception)
|
78
|
+
requeueable_exceptions.member?(exception)
|
79
|
+
end
|
80
|
+
|
81
|
+
def requeueable_exceptions
|
82
|
+
@requeueable_exceptions ||= {}
|
83
|
+
end
|
84
|
+
|
85
|
+
def requeuable_exception_for(e)
|
86
|
+
requeueable_exceptions.fetch(e.class) do
|
87
|
+
requeueable_exceptions.each do |klass, exc|
|
88
|
+
break exc if klass === e
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -1,13 +1,24 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
1
3
|
module Qless
|
2
4
|
module Middleware
|
5
|
+
# Auto-retries particular errors using qless-core's internal retry tracking
|
6
|
+
# mechanism. Supports a backoff strategy (typically exponential).
|
7
|
+
#
|
8
|
+
# Note: this does not support varying the number of allowed retries by
|
9
|
+
# exception type. If you want that kind of flexibility, use the
|
10
|
+
# RequeueExceptions middleware instead.
|
3
11
|
module RetryExceptions
|
4
12
|
def around_perform(job)
|
5
13
|
super
|
6
|
-
rescue *retryable_exception_classes =>
|
14
|
+
rescue *retryable_exception_classes => error
|
7
15
|
raise if job.retries_left <= 0
|
8
16
|
|
9
17
|
attempt_num = (job.original_retries - job.retries_left) + 1
|
10
|
-
|
18
|
+
failure = Qless.failure_formatter.format(job, error)
|
19
|
+
job.retry(backoff_strategy.call(attempt_num, error), *failure)
|
20
|
+
|
21
|
+
on_retry_callback.call(error, job)
|
11
22
|
end
|
12
23
|
|
13
24
|
def retryable_exception_classes
|
@@ -18,7 +29,7 @@ module Qless
|
|
18
29
|
retryable_exception_classes.push(*exception_classes)
|
19
30
|
end
|
20
31
|
|
21
|
-
NO_BACKOFF_STRATEGY =
|
32
|
+
NO_BACKOFF_STRATEGY = ->(_num, _error) { 0 }
|
22
33
|
|
23
34
|
def use_backoff_strategy(strategy = nil, &block)
|
24
35
|
@backoff_strategy = strategy || block
|
@@ -28,16 +39,34 @@ module Qless
|
|
28
39
|
@backoff_strategy ||= NO_BACKOFF_STRATEGY
|
29
40
|
end
|
30
41
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
42
|
+
DEFAULT_ON_RETRY_CALLBACK = lambda { |error, job| }
|
43
|
+
def use_on_retry_callback(&block)
|
44
|
+
@on_retry_callback = block if block
|
45
|
+
end
|
46
|
+
|
47
|
+
def on_retry_callback
|
48
|
+
@on_retry_callback ||= DEFAULT_ON_RETRY_CALLBACK
|
49
|
+
end
|
50
|
+
|
51
|
+
# If `factor` is omitted it is set to `delay_seconds` to reproduce legacy
|
52
|
+
# behavior.
|
53
|
+
def exponential(delay_seconds, options={})
|
54
|
+
factor = options.fetch(:factor, delay_seconds)
|
55
|
+
fuzz_factor = options.fetch(:fuzz_factor, 0)
|
56
|
+
|
57
|
+
lambda do |retry_no, error|
|
58
|
+
unfuzzed = delay_seconds * factor**(retry_no - 1)
|
59
|
+
return unfuzzed if fuzz_factor.zero?
|
60
|
+
r = 2 * rand - 1
|
61
|
+
# r is uniformly distributed in range [-1, 1]
|
62
|
+
unfuzzed * (1 + fuzz_factor * r)
|
35
63
|
end
|
36
64
|
end
|
37
65
|
end
|
38
66
|
end
|
39
67
|
|
40
68
|
# For backwards compatibility
|
41
|
-
|
69
|
+
module RetryExceptions
|
70
|
+
include Middleware::RetryExceptions
|
71
|
+
end
|
42
72
|
end
|
43
|
-
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
1
3
|
require 'raven'
|
2
4
|
|
3
5
|
module Qless
|
@@ -37,7 +39,6 @@ module Qless
|
|
37
39
|
# Qless Web UI.
|
38
40
|
end
|
39
41
|
|
40
|
-
|
41
42
|
def job_metadata
|
42
43
|
{
|
43
44
|
jid: @job.jid,
|
@@ -55,11 +56,7 @@ module Qless
|
|
55
56
|
def job_history
|
56
57
|
@job.queue_history.map do |history_event|
|
57
58
|
history_event.each_with_object({}) do |(key, value), hash|
|
58
|
-
hash[key] =
|
59
|
-
value.iso8601
|
60
|
-
else
|
61
|
-
value
|
62
|
-
end
|
59
|
+
hash[key] = value.is_a?(Time) ? value.iso8601 : value
|
63
60
|
end
|
64
61
|
end
|
65
62
|
end
|
@@ -67,4 +64,3 @@ module Qless
|
|
67
64
|
end
|
68
65
|
end
|
69
66
|
end
|
70
|
-
|
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'qless/middleware/requeue_exceptions'
|
3
|
+
|
4
|
+
module Qless
|
5
|
+
# Unique error class used when a job is timed out by this middleware.
|
6
|
+
# Allows us to differentiate this timeout from others caused by `::Timeout::Erorr`
|
7
|
+
JobTimedoutError = Class.new(StandardError)
|
8
|
+
InvalidTimeoutError = Class.new(ArgumentError)
|
9
|
+
|
10
|
+
module Middleware
|
11
|
+
# Applies a hard time out. To use this middleware, instantiate it and pass a block; the block
|
12
|
+
# will be passed the job object (which has a `ttl` method for getting the job's remaining TTL),
|
13
|
+
# and the block should return the desired timeout in seconds.
|
14
|
+
# This allows you to set a hard constant time out to a particular job class
|
15
|
+
# (using something like `extend Qless::Middleware::Timeout.new { 60 * 60 }`),
|
16
|
+
# or a variable timeout based on the individual TTLs of each job
|
17
|
+
# (using something like `extend Qless::Middleware::Timeout.new { |job| job.ttl * 1.1 }`).
|
18
|
+
class Timeout < Module
|
19
|
+
def initialize(opts = {})
|
20
|
+
timeout_class = opts.fetch(:timeout_class, ::Timeout)
|
21
|
+
kernel_class = opts.fetch(:kernel_class, Kernel)
|
22
|
+
module_eval do
|
23
|
+
define_method :around_perform do |job|
|
24
|
+
timeout_seconds = yield job
|
25
|
+
|
26
|
+
return super(job) if timeout_seconds.nil?
|
27
|
+
|
28
|
+
if !timeout_seconds.is_a?(Numeric) || timeout_seconds <= 0
|
29
|
+
raise InvalidTimeoutError, "Timeout must be a positive number or nil, " \
|
30
|
+
"but was #{timeout_seconds}"
|
31
|
+
end
|
32
|
+
|
33
|
+
begin
|
34
|
+
timeout_class.timeout(timeout_seconds) { super(job) }
|
35
|
+
rescue ::Timeout::Error => e
|
36
|
+
error = JobTimedoutError.new("Qless: job timeout (#{timeout_seconds}) exceeded.")
|
37
|
+
error.set_backtrace(e.backtrace)
|
38
|
+
# The stalled connection to redis might be the cause of the timeout. We cannot rely
|
39
|
+
# on state of connection either (e.g., we might be in the middle of Redis call when
|
40
|
+
# timeout happend). To play it safe, we reconnect.
|
41
|
+
job.reconnect_to_redis
|
42
|
+
job.fail(*Qless.failure_formatter.format(job, error, []))
|
43
|
+
# Since we are leaving with bang (exit!), normal requeue logic does not work.
|
44
|
+
# Do it manually right here.
|
45
|
+
if self.is_a?(::Qless::Middleware::RequeueExceptions) &&
|
46
|
+
self.requeueable?(JobTimedoutError)
|
47
|
+
self.handle_exception(job, error)
|
48
|
+
end
|
49
|
+
|
50
|
+
# ::Timeout.timeout is dangerous to use as it can leave things in an inconsistent
|
51
|
+
# state. With Redis, for example, we've seen the socket buffer left with unread bytes
|
52
|
+
# on it, which can affect later redis calls. Thus, it's much safer just to exit, and
|
53
|
+
# allow the parent process to restart the worker in a known, clean state.
|
54
|
+
#
|
55
|
+
# We use 73 as a unique exit status for this case. 73 looks
|
56
|
+
# a bit like TE (Timeout::Error)
|
57
|
+
kernel_class.exit!(73)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|