reqless 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +8 -0
- data/README.md +648 -0
- data/Rakefile +117 -0
- data/bin/docker-build-and-test +22 -0
- data/exe/reqless-web +11 -0
- data/lib/reqless/config.rb +31 -0
- data/lib/reqless/failure_formatter.rb +43 -0
- data/lib/reqless/job.rb +496 -0
- data/lib/reqless/job_reservers/ordered.rb +29 -0
- data/lib/reqless/job_reservers/round_robin.rb +46 -0
- data/lib/reqless/job_reservers/shuffled_round_robin.rb +21 -0
- data/lib/reqless/lua/reqless-lib.lua +2965 -0
- data/lib/reqless/lua/reqless.lua +2545 -0
- data/lib/reqless/lua_script.rb +90 -0
- data/lib/reqless/middleware/requeue_exceptions.rb +94 -0
- data/lib/reqless/middleware/retry_exceptions.rb +72 -0
- data/lib/reqless/middleware/sentry.rb +66 -0
- data/lib/reqless/middleware/timeout.rb +63 -0
- data/lib/reqless/queue.rb +189 -0
- data/lib/reqless/queue_priority_pattern.rb +16 -0
- data/lib/reqless/server/static/css/bootstrap-responsive.css +686 -0
- data/lib/reqless/server/static/css/bootstrap-responsive.min.css +12 -0
- data/lib/reqless/server/static/css/bootstrap.css +3991 -0
- data/lib/reqless/server/static/css/bootstrap.min.css +689 -0
- data/lib/reqless/server/static/css/codemirror.css +112 -0
- data/lib/reqless/server/static/css/docs.css +839 -0
- data/lib/reqless/server/static/css/jquery.noty.css +105 -0
- data/lib/reqless/server/static/css/noty_theme_twitter.css +137 -0
- data/lib/reqless/server/static/css/style.css +200 -0
- data/lib/reqless/server/static/favicon.ico +0 -0
- data/lib/reqless/server/static/img/glyphicons-halflings-white.png +0 -0
- data/lib/reqless/server/static/img/glyphicons-halflings.png +0 -0
- data/lib/reqless/server/static/js/bootstrap-alert.js +94 -0
- data/lib/reqless/server/static/js/bootstrap-scrollspy.js +125 -0
- data/lib/reqless/server/static/js/bootstrap-tab.js +130 -0
- data/lib/reqless/server/static/js/bootstrap-tooltip.js +270 -0
- data/lib/reqless/server/static/js/bootstrap-typeahead.js +285 -0
- data/lib/reqless/server/static/js/bootstrap.js +1726 -0
- data/lib/reqless/server/static/js/bootstrap.min.js +6 -0
- data/lib/reqless/server/static/js/codemirror.js +2972 -0
- data/lib/reqless/server/static/js/jquery.noty.js +220 -0
- data/lib/reqless/server/static/js/mode/javascript.js +360 -0
- data/lib/reqless/server/static/js/theme/cobalt.css +18 -0
- data/lib/reqless/server/static/js/theme/eclipse.css +25 -0
- data/lib/reqless/server/static/js/theme/elegant.css +10 -0
- data/lib/reqless/server/static/js/theme/lesser-dark.css +45 -0
- data/lib/reqless/server/static/js/theme/monokai.css +28 -0
- data/lib/reqless/server/static/js/theme/neat.css +9 -0
- data/lib/reqless/server/static/js/theme/night.css +21 -0
- data/lib/reqless/server/static/js/theme/rubyblue.css +21 -0
- data/lib/reqless/server/static/js/theme/xq-dark.css +46 -0
- data/lib/reqless/server/views/_job.erb +259 -0
- data/lib/reqless/server/views/_job_list.erb +8 -0
- data/lib/reqless/server/views/_pagination.erb +7 -0
- data/lib/reqless/server/views/about.erb +130 -0
- data/lib/reqless/server/views/completed.erb +11 -0
- data/lib/reqless/server/views/config.erb +14 -0
- data/lib/reqless/server/views/failed.erb +48 -0
- data/lib/reqless/server/views/failed_type.erb +18 -0
- data/lib/reqless/server/views/job.erb +17 -0
- data/lib/reqless/server/views/layout.erb +451 -0
- data/lib/reqless/server/views/overview.erb +137 -0
- data/lib/reqless/server/views/queue.erb +125 -0
- data/lib/reqless/server/views/queues.erb +45 -0
- data/lib/reqless/server/views/tag.erb +6 -0
- data/lib/reqless/server/views/throttles.erb +38 -0
- data/lib/reqless/server/views/track.erb +75 -0
- data/lib/reqless/server/views/worker.erb +34 -0
- data/lib/reqless/server/views/workers.erb +14 -0
- data/lib/reqless/server.rb +549 -0
- data/lib/reqless/subscriber.rb +74 -0
- data/lib/reqless/test_helpers/worker_helpers.rb +55 -0
- data/lib/reqless/throttle.rb +57 -0
- data/lib/reqless/version.rb +5 -0
- data/lib/reqless/worker/base.rb +237 -0
- data/lib/reqless/worker/forking.rb +215 -0
- data/lib/reqless/worker/serial.rb +41 -0
- data/lib/reqless/worker.rb +5 -0
- data/lib/reqless.rb +309 -0
- metadata +399 -0
@@ -0,0 +1,90 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'digest/sha1'
|
4
|
+
|
5
|
+
module Reqless
|
6
|
+
LuaScriptError = Class.new(Reqless::Error)
|
7
|
+
|
8
|
+
# Wraps a lua script. Knows how to reload it if necessary
|
9
|
+
class LuaScript
|
10
|
+
DEFAULT_ON_RELOAD_CALLBACK = proc {}
|
11
|
+
SCRIPT_ROOT = File.expand_path('../lua', __FILE__)
|
12
|
+
|
13
|
+
def initialize(name, redis, options = {})
|
14
|
+
@name = name
|
15
|
+
@on_reload_callback = options[:on_reload_callback] || DEFAULT_ON_RELOAD_CALLBACK
|
16
|
+
@redis = redis
|
17
|
+
@sha = Digest::SHA1.hexdigest(script_contents)
|
18
|
+
end
|
19
|
+
|
20
|
+
attr_reader :name, :redis, :sha
|
21
|
+
|
22
|
+
def reload
|
23
|
+
@sha = @redis.script(:load, script_contents)
|
24
|
+
@on_reload_callback.call(@redis, @sha)
|
25
|
+
@sha
|
26
|
+
end
|
27
|
+
|
28
|
+
def call(*argv)
|
29
|
+
handle_no_script_error do
|
30
|
+
_call(*argv)
|
31
|
+
end
|
32
|
+
rescue Redis::CommandError => err
|
33
|
+
if match = err.message.match('user_script:\d+:\s*(\w+.+$)')
|
34
|
+
raise LuaScriptError.new(match[1])
|
35
|
+
else
|
36
|
+
raise err
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
def _call(*argv)
|
42
|
+
@redis.evalsha(@sha, keys: [], argv: argv)
|
43
|
+
end
|
44
|
+
|
45
|
+
def handle_no_script_error
|
46
|
+
yield
|
47
|
+
rescue ScriptNotLoadedRedisCommandError
|
48
|
+
reload
|
49
|
+
yield
|
50
|
+
end
|
51
|
+
|
52
|
+
# Module for notifying when a script hasn't yet been loaded
|
53
|
+
module ScriptNotLoadedRedisCommandError
|
54
|
+
MESSAGE = 'NOSCRIPT No matching script. Please use EVAL.'
|
55
|
+
|
56
|
+
def self.===(error)
|
57
|
+
error.is_a?(Redis::CommandError) && error.message.include?(MESSAGE)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def script_contents
|
62
|
+
@script_contents ||= File.read(File.join(SCRIPT_ROOT, "#{@name}.lua"))
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
# Provides a simple way to load and use lua-based Reqless plugins.
|
67
|
+
# This combines the reqless-lib.lua script plus your custom script
|
68
|
+
# contents all into one script, so that your script can use
|
69
|
+
# Reqless's lua API.
|
70
|
+
class LuaPlugin < LuaScript
|
71
|
+
def initialize(name, redis, plugin_contents)
|
72
|
+
@name = name
|
73
|
+
@redis = redis
|
74
|
+
@plugin_contents = plugin_contents.gsub(COMMENT_LINES_RE, '')
|
75
|
+
super(name, redis)
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
|
80
|
+
def script_contents
|
81
|
+
@script_contents ||= [REQLESS_LIB_CONTENTS, @plugin_contents].join("\n\n")
|
82
|
+
end
|
83
|
+
|
84
|
+
COMMENT_LINES_RE = /^\s*--.*$\n?/
|
85
|
+
|
86
|
+
REQLESS_LIB_CONTENTS = File.read(
|
87
|
+
File.join(SCRIPT_ROOT, 'reqless-lib.lua')
|
88
|
+
).gsub(COMMENT_LINES_RE, '')
|
89
|
+
end
|
90
|
+
end
|
@@ -0,0 +1,94 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
module Reqless
|
4
|
+
module Middleware
|
5
|
+
# This middleware is like RetryExceptions, but it doesn't use reqless-core's
|
6
|
+
# internal retry/retry-tracking mechanism. Instead, it re-queues the job
|
7
|
+
# when it fails with a matched error, and increments a counter in the job's
|
8
|
+
# data.
|
9
|
+
#
|
10
|
+
# This is useful for exceptions for which you want a different
|
11
|
+
# backoff/retry strategy. The internal retry mechanism doesn't allow for
|
12
|
+
# separate tracking by exception type, and thus doesn't allow you to retry
|
13
|
+
# different exceptions a different number of times.
|
14
|
+
#
|
15
|
+
# This is particularly useful for handling resource throttling errors,
|
16
|
+
# where you may not want exponential backoff, and you may want the error
|
17
|
+
# to be retried many times, w/o having other transient errors retried so
|
18
|
+
# many times.
|
19
|
+
module RequeueExceptions
|
20
|
+
RequeueableException = Struct.new(:klass, :delay_min, :delay_span, :max_attempts) do
|
21
|
+
def self.from_splat_and_options(*klasses, options)
|
22
|
+
delay_range = options.fetch(:delay_range)
|
23
|
+
delay_min = Float(delay_range.min)
|
24
|
+
delay_span = Float(delay_range.max) - Float(delay_range.min)
|
25
|
+
max_attempts = options.fetch(:max_attempts)
|
26
|
+
klasses.map do |klass|
|
27
|
+
new(klass, delay_min, delay_span, max_attempts)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def delay
|
32
|
+
delay_min + Random.rand(delay_span)
|
33
|
+
end
|
34
|
+
|
35
|
+
def raise_if_exhausted_requeues(error, requeues)
|
36
|
+
raise error if requeues >= max_attempts
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def requeue_on(*exceptions, options)
|
41
|
+
RequeueableException.from_splat_and_options(
|
42
|
+
*exceptions, options).each do |exc|
|
43
|
+
requeueable_exceptions[exc.klass] = exc
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
DEFAULT_ON_REQUEUE_CALLBACK = lambda { |error, job| }
|
48
|
+
def use_on_requeue_callback(&block)
|
49
|
+
@on_requeue_callback = block if block
|
50
|
+
end
|
51
|
+
|
52
|
+
def on_requeue_callback
|
53
|
+
@on_requeue_callback ||= DEFAULT_ON_REQUEUE_CALLBACK
|
54
|
+
end
|
55
|
+
|
56
|
+
def handle_exception(job, error)
|
57
|
+
config = requeuable_exception_for(error)
|
58
|
+
|
59
|
+
requeues_by_exception = (job.data['requeues_by_exception'] ||= {})
|
60
|
+
requeues_by_exception[config.klass.name] ||= 0
|
61
|
+
|
62
|
+
config.raise_if_exhausted_requeues(
|
63
|
+
error, requeues_by_exception[config.klass.name])
|
64
|
+
|
65
|
+
requeues_by_exception[config.klass.name] += 1
|
66
|
+
job.requeue(job.queue_name, delay: config.delay, data: job.data)
|
67
|
+
|
68
|
+
on_requeue_callback.call(error, job)
|
69
|
+
end
|
70
|
+
|
71
|
+
def around_perform(job)
|
72
|
+
super
|
73
|
+
rescue *requeueable_exceptions.keys => e
|
74
|
+
handle_exception(job, e)
|
75
|
+
end
|
76
|
+
|
77
|
+
def requeueable?(exception)
|
78
|
+
requeueable_exceptions.member?(exception)
|
79
|
+
end
|
80
|
+
|
81
|
+
def requeueable_exceptions
|
82
|
+
@requeueable_exceptions ||= {}
|
83
|
+
end
|
84
|
+
|
85
|
+
def requeuable_exception_for(e)
|
86
|
+
requeueable_exceptions.fetch(e.class) do
|
87
|
+
requeueable_exceptions.each do |klass, exc|
|
88
|
+
break exc if klass === e
|
89
|
+
end
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
@@ -0,0 +1,72 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
module Reqless
|
4
|
+
module Middleware
|
5
|
+
# Auto-retries particular errors using reqless-core's internal retry tracking
|
6
|
+
# mechanism. Supports a backoff strategy (typically exponential).
|
7
|
+
#
|
8
|
+
# Note: this does not support varying the number of allowed retries by
|
9
|
+
# exception type. If you want that kind of flexibility, use the
|
10
|
+
# RequeueExceptions middleware instead.
|
11
|
+
module RetryExceptions
|
12
|
+
def around_perform(job)
|
13
|
+
super
|
14
|
+
rescue *retryable_exception_classes => error
|
15
|
+
raise if job.retries_left <= 0
|
16
|
+
|
17
|
+
attempt_num = (job.original_retries - job.retries_left) + 1
|
18
|
+
failure = Reqless.failure_formatter.format(job, error)
|
19
|
+
job.retry(backoff_strategy.call(attempt_num, error), *failure)
|
20
|
+
|
21
|
+
on_retry_callback.call(error, job)
|
22
|
+
end
|
23
|
+
|
24
|
+
def retryable_exception_classes
|
25
|
+
@retryable_exception_classes ||= []
|
26
|
+
end
|
27
|
+
|
28
|
+
def retry_on(*exception_classes)
|
29
|
+
retryable_exception_classes.push(*exception_classes)
|
30
|
+
end
|
31
|
+
|
32
|
+
NO_BACKOFF_STRATEGY = ->(_num, _error) { 0 }
|
33
|
+
|
34
|
+
def use_backoff_strategy(strategy = nil, &block)
|
35
|
+
@backoff_strategy = strategy || block
|
36
|
+
end
|
37
|
+
|
38
|
+
def backoff_strategy
|
39
|
+
@backoff_strategy ||= NO_BACKOFF_STRATEGY
|
40
|
+
end
|
41
|
+
|
42
|
+
DEFAULT_ON_RETRY_CALLBACK = lambda { |error, job| }
|
43
|
+
def use_on_retry_callback(&block)
|
44
|
+
@on_retry_callback = block if block
|
45
|
+
end
|
46
|
+
|
47
|
+
def on_retry_callback
|
48
|
+
@on_retry_callback ||= DEFAULT_ON_RETRY_CALLBACK
|
49
|
+
end
|
50
|
+
|
51
|
+
# If `factor` is omitted it is set to `delay_seconds` to reproduce legacy
|
52
|
+
# behavior.
|
53
|
+
def exponential(delay_seconds, options={})
|
54
|
+
factor = options.fetch(:factor, delay_seconds)
|
55
|
+
fuzz_factor = options.fetch(:fuzz_factor, 0)
|
56
|
+
|
57
|
+
lambda do |retry_no, error|
|
58
|
+
unfuzzed = delay_seconds * factor**(retry_no - 1)
|
59
|
+
return unfuzzed if fuzz_factor.zero?
|
60
|
+
r = 2 * rand - 1
|
61
|
+
# r is uniformly distributed in range [-1, 1]
|
62
|
+
unfuzzed * (1 + fuzz_factor * r)
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
# For backwards compatibility
|
69
|
+
module RetryExceptions
|
70
|
+
include Middleware::RetryExceptions
|
71
|
+
end
|
72
|
+
end
|
@@ -0,0 +1,66 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'raven'
|
4
|
+
|
5
|
+
module Reqless
|
6
|
+
module Middleware
|
7
|
+
# This middleware logs errors to the sentry exception notification service:
|
8
|
+
# http://getsentry.com/
|
9
|
+
module Sentry
|
10
|
+
def around_perform(job)
|
11
|
+
super
|
12
|
+
rescue Exception => e
|
13
|
+
SentryLogger.new(e, job).log
|
14
|
+
raise
|
15
|
+
end
|
16
|
+
|
17
|
+
# Logs a single exception to Sentry, adding pertinent job info.
|
18
|
+
class SentryLogger
|
19
|
+
def initialize(exception, job)
|
20
|
+
@exception, @job = exception, job
|
21
|
+
end
|
22
|
+
|
23
|
+
def log
|
24
|
+
event = ::Raven::Event.capture_exception(@exception) do |evt|
|
25
|
+
evt.extra = { job: job_metadata }
|
26
|
+
end
|
27
|
+
|
28
|
+
safely_send event
|
29
|
+
end
|
30
|
+
|
31
|
+
private
|
32
|
+
|
33
|
+
def safely_send(event)
|
34
|
+
return unless event
|
35
|
+
::Raven.send(event)
|
36
|
+
rescue
|
37
|
+
# We don't want to silence our errors when the Sentry server
|
38
|
+
# responds with an error. We'll still see the errors on the
|
39
|
+
# Reqless Web UI.
|
40
|
+
end
|
41
|
+
|
42
|
+
def job_metadata
|
43
|
+
{
|
44
|
+
jid: @job.jid,
|
45
|
+
klass: @job.klass_name,
|
46
|
+
history: job_history,
|
47
|
+
data: @job.data,
|
48
|
+
queue: @job.queue_name,
|
49
|
+
worker: @job.worker_name,
|
50
|
+
tags: @job.tags,
|
51
|
+
priority: @job.priority
|
52
|
+
}
|
53
|
+
end
|
54
|
+
|
55
|
+
# We want to log formatted timestamps rather than integer timestamps
|
56
|
+
def job_history
|
57
|
+
@job.queue_history.map do |history_event|
|
58
|
+
history_event.each_with_object({}) do |(key, value), hash|
|
59
|
+
hash[key] = value.is_a?(Time) ? value.iso8601 : value
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
require 'reqless/middleware/requeue_exceptions'
|
3
|
+
|
4
|
+
module Reqless
|
5
|
+
# Unique error class used when a job is timed out by this middleware.
|
6
|
+
# Allows us to differentiate this timeout from others caused by `::Timeout::Erorr`
|
7
|
+
JobTimedoutError = Class.new(StandardError)
|
8
|
+
InvalidTimeoutError = Class.new(ArgumentError)
|
9
|
+
|
10
|
+
module Middleware
|
11
|
+
# Applies a hard time out. To use this middleware, instantiate it and pass a block; the block
|
12
|
+
# will be passed the job object (which has a `ttl` method for getting the job's remaining TTL),
|
13
|
+
# and the block should return the desired timeout in seconds.
|
14
|
+
# This allows you to set a hard constant time out to a particular job class
|
15
|
+
# (using something like `extend Reqless::Middleware::Timeout.new { 60 * 60 }`),
|
16
|
+
# or a variable timeout based on the individual TTLs of each job
|
17
|
+
# (using something like `extend Reqless::Middleware::Timeout.new { |job| job.ttl * 1.1 }`).
|
18
|
+
class Timeout < Module
|
19
|
+
def initialize(opts = {})
|
20
|
+
timeout_class = opts.fetch(:timeout_class, ::Timeout)
|
21
|
+
kernel_class = opts.fetch(:kernel_class, Kernel)
|
22
|
+
module_eval do
|
23
|
+
define_method :around_perform do |job|
|
24
|
+
timeout_seconds = yield job
|
25
|
+
|
26
|
+
return super(job) if timeout_seconds.nil?
|
27
|
+
|
28
|
+
if !timeout_seconds.is_a?(Numeric) || timeout_seconds <= 0
|
29
|
+
raise InvalidTimeoutError, "Timeout must be a positive number or nil, " \
|
30
|
+
"but was #{timeout_seconds}"
|
31
|
+
end
|
32
|
+
|
33
|
+
begin
|
34
|
+
timeout_class.timeout(timeout_seconds) { super(job) }
|
35
|
+
rescue ::Timeout::Error => e
|
36
|
+
error = JobTimedoutError.new(e.message)
|
37
|
+
error.set_backtrace(e.backtrace)
|
38
|
+
# The stalled connection to redis might be the cause of the timeout. We cannot rely
|
39
|
+
# on state of connection either (e.g., we might be in the middle of Redis call when
|
40
|
+
# timeout happend). To play it safe, we reconnect.
|
41
|
+
job.fail(*Reqless.failure_formatter.format(job, error, []))
|
42
|
+
# Since we are leaving with bang (exit!), normal requeue logic does not work.
|
43
|
+
# Do it manually right here.
|
44
|
+
if self.is_a?(::Reqless::Middleware::RequeueExceptions) &&
|
45
|
+
self.requeueable?(JobTimedoutError)
|
46
|
+
self.handle_exception(job, error)
|
47
|
+
end
|
48
|
+
|
49
|
+
# ::Timeout.timeout is dangerous to use as it can leave things in an inconsistent
|
50
|
+
# state. With Redis, for example, we've seen the socket buffer left with unread bytes
|
51
|
+
# on it, which can affect later redis calls. Thus, it's much safer just to exit, and
|
52
|
+
# allow the parent process to restart the worker in a known, clean state.
|
53
|
+
#
|
54
|
+
# We use 73 as a unique exit status for this case. 73 looks
|
55
|
+
# a bit like TE (Timeout::Error)
|
56
|
+
kernel_class.exit!(73)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,189 @@
|
|
1
|
+
# Encoding: utf-8
|
2
|
+
|
3
|
+
require 'reqless/job'
|
4
|
+
require 'redis'
|
5
|
+
require 'json'
|
6
|
+
|
7
|
+
module Reqless
|
8
|
+
# A class for interacting with jobs in different states in a queue. Not meant
|
9
|
+
# to be instantiated directly, it's accessed with Queue#jobs
|
10
|
+
class QueueJobs
|
11
|
+
def initialize(name, client)
|
12
|
+
@name = name
|
13
|
+
@client = client
|
14
|
+
end
|
15
|
+
|
16
|
+
def running(start = 0, count = 25)
|
17
|
+
JSON.parse(@client.call('queue.jobsByState', 'running', @name, start, count))
|
18
|
+
end
|
19
|
+
|
20
|
+
def throttled(start = 0, count = 25)
|
21
|
+
JSON.parse(@client.call('queue.jobsByState', 'throttled', @name, start, count))
|
22
|
+
end
|
23
|
+
|
24
|
+
def stalled(start = 0, count = 25)
|
25
|
+
JSON.parse(@client.call('queue.jobsByState', 'stalled', @name, start, count))
|
26
|
+
end
|
27
|
+
|
28
|
+
def scheduled(start = 0, count = 25)
|
29
|
+
JSON.parse(@client.call('queue.jobsByState', 'scheduled', @name, start, count))
|
30
|
+
end
|
31
|
+
|
32
|
+
def depends(start = 0, count = 25)
|
33
|
+
JSON.parse(@client.call('queue.jobsByState', 'depends', @name, start, count))
|
34
|
+
end
|
35
|
+
|
36
|
+
def recurring(start = 0, count = 25)
|
37
|
+
JSON.parse(@client.call('queue.jobsByState', 'recurring', @name, start, count))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# A class for interacting with a specific queue. Not meant to be instantiated
|
42
|
+
# directly, it's accessed with Client#queues[...]
|
43
|
+
class Queue
|
44
|
+
attr_reader :name, :client
|
45
|
+
|
46
|
+
def initialize(name, client)
|
47
|
+
@client = client
|
48
|
+
@name = name
|
49
|
+
end
|
50
|
+
|
51
|
+
# Our worker name is the same as our client's
|
52
|
+
def worker_name
|
53
|
+
@client.worker_name
|
54
|
+
end
|
55
|
+
|
56
|
+
def jobs
|
57
|
+
@jobs ||= QueueJobs.new(@name, @client)
|
58
|
+
end
|
59
|
+
|
60
|
+
def counts
|
61
|
+
JSON.parse(@client.call('queue.counts', @name))
|
62
|
+
end
|
63
|
+
|
64
|
+
def heartbeat
|
65
|
+
get_config :heartbeat
|
66
|
+
end
|
67
|
+
|
68
|
+
def heartbeat=(value)
|
69
|
+
set_config :heartbeat, value
|
70
|
+
end
|
71
|
+
|
72
|
+
def throttle
|
73
|
+
@throttle ||= Reqless::Throttle.new("ql:q:#{name}", client)
|
74
|
+
end
|
75
|
+
|
76
|
+
def paused?
|
77
|
+
counts['paused']
|
78
|
+
end
|
79
|
+
|
80
|
+
def pause(opts = {})
|
81
|
+
@client.call('queue.pause', name)
|
82
|
+
@client.call('job.timeout', jobs.running(0, -1)) unless opts[:stopjobs].nil?
|
83
|
+
end
|
84
|
+
|
85
|
+
def unpause
|
86
|
+
@client.call('queue.unpause', name)
|
87
|
+
end
|
88
|
+
|
89
|
+
# Put the described job in this queue
|
90
|
+
# Options include:
|
91
|
+
# => priority (int)
|
92
|
+
# => tags (array of strings)
|
93
|
+
# => delay (int)
|
94
|
+
# => throttles (array of strings)
|
95
|
+
def put(klass, data, opts = {})
|
96
|
+
opts = job_options(klass, data, opts)
|
97
|
+
@client.call(
|
98
|
+
'queue.put',
|
99
|
+
worker_name, @name,
|
100
|
+
(opts[:jid] || Reqless.generate_jid),
|
101
|
+
klass.is_a?(String) ? klass : klass.name,
|
102
|
+
*Job.build_opts_array(opts.merge(:data => data)),
|
103
|
+
)
|
104
|
+
end
|
105
|
+
|
106
|
+
# Make a recurring job in this queue
|
107
|
+
# Options include:
|
108
|
+
# => priority (int)
|
109
|
+
# => tags (array of strings)
|
110
|
+
# => retries (int)
|
111
|
+
# => offset (int)
|
112
|
+
def recur(klass, data, interval, opts = {})
|
113
|
+
opts = job_options(klass, data, opts)
|
114
|
+
@client.call(
|
115
|
+
'queue.recurAtInterval',
|
116
|
+
@name,
|
117
|
+
(opts[:jid] || Reqless.generate_jid),
|
118
|
+
klass.is_a?(String) ? klass : klass.name,
|
119
|
+
JSON.generate(data),
|
120
|
+
interval, opts.fetch(:offset, 0),
|
121
|
+
'priority', opts.fetch(:priority, 0),
|
122
|
+
'tags', JSON.generate(opts.fetch(:tags, [])),
|
123
|
+
'retries', opts.fetch(:retries, 5),
|
124
|
+
'backlog', opts.fetch(:backlog, 0)
|
125
|
+
)
|
126
|
+
end
|
127
|
+
|
128
|
+
# Pop a work item off the queue
|
129
|
+
def pop(count = nil)
|
130
|
+
jids = JSON.parse(@client.call('queue.pop', @name, worker_name, (count || 1)))
|
131
|
+
jobs = jids.map { |j| Job.new(@client, j) }
|
132
|
+
count.nil? ? jobs[0] : jobs
|
133
|
+
end
|
134
|
+
|
135
|
+
# Peek at a work item
|
136
|
+
def peek(offset_or_count = nil, count = nil)
|
137
|
+
actual_offset = offset_or_count && count ? offset_or_count : 0
|
138
|
+
actual_count = offset_or_count && count ? count : (offset_or_count || 1)
|
139
|
+
return_single_job = offset_or_count.nil? && count.nil?
|
140
|
+
jids = JSON.parse(@client.call('queue.peek', @name, actual_offset, actual_count))
|
141
|
+
jobs = jids.map { |j| Job.new(@client, j) }
|
142
|
+
return_single_job ? jobs[0] : jobs
|
143
|
+
end
|
144
|
+
|
145
|
+
def stats(date = nil)
|
146
|
+
JSON.parse(@client.call('queue.stats', @name, (date || Time.now.to_f)))
|
147
|
+
end
|
148
|
+
|
149
|
+
# How many items in the queue?
|
150
|
+
def length
|
151
|
+
(@client.redis.multi do |pipeline|
|
152
|
+
pipeline.zcard("ql:q:#{@name}-locks")
|
153
|
+
pipeline.zcard("ql:q:#{@name}-work")
|
154
|
+
pipeline.zcard("ql:q:#{@name}-scheduled")
|
155
|
+
end).inject(0, :+)
|
156
|
+
end
|
157
|
+
|
158
|
+
def to_s
|
159
|
+
"#<Reqless::Queue #{@name}>"
|
160
|
+
end
|
161
|
+
alias_method :inspect, :to_s
|
162
|
+
|
163
|
+
def ==(other)
|
164
|
+
self.class == other.class &&
|
165
|
+
client == other.client &&
|
166
|
+
name.to_s == other.name.to_s
|
167
|
+
end
|
168
|
+
alias eql? ==
|
169
|
+
|
170
|
+
def hash
|
171
|
+
self.class.hash ^ client.hash ^ name.to_s.hash
|
172
|
+
end
|
173
|
+
|
174
|
+
private
|
175
|
+
|
176
|
+
def job_options(klass, data, opts)
|
177
|
+
return opts unless klass.respond_to?(:default_job_options)
|
178
|
+
klass.default_job_options(data).merge(opts)
|
179
|
+
end
|
180
|
+
|
181
|
+
def set_config(config, value)
|
182
|
+
@client.config["#{@name}-#{config}"] = value
|
183
|
+
end
|
184
|
+
|
185
|
+
def get_config(config)
|
186
|
+
@client.config["#{@name}-#{config}"]
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module Reqless
|
2
|
+
class QueuePriorityPattern
|
3
|
+
attr_reader :pattern, :should_distribute_fairly
|
4
|
+
|
5
|
+
def initialize(pattern, should_distribute_fairly = false)
|
6
|
+
@pattern = pattern
|
7
|
+
@should_distribute_fairly = should_distribute_fairly
|
8
|
+
end
|
9
|
+
|
10
|
+
def ==(other)
|
11
|
+
return self.class == other.class &&
|
12
|
+
self.pattern.join == other.pattern.join &&
|
13
|
+
self.should_distribute_fairly == other.should_distribute_fairly
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|