qless 0.9.3 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. data/Gemfile +9 -3
  2. data/README.md +70 -25
  3. data/Rakefile +125 -9
  4. data/exe/install_phantomjs +21 -0
  5. data/lib/qless.rb +115 -76
  6. data/lib/qless/config.rb +11 -9
  7. data/lib/qless/failure_formatter.rb +43 -0
  8. data/lib/qless/job.rb +201 -102
  9. data/lib/qless/job_reservers/ordered.rb +7 -1
  10. data/lib/qless/job_reservers/round_robin.rb +16 -6
  11. data/lib/qless/job_reservers/shuffled_round_robin.rb +9 -2
  12. data/lib/qless/lua/qless-lib.lua +2463 -0
  13. data/lib/qless/lua/qless.lua +2012 -0
  14. data/lib/qless/lua_script.rb +63 -12
  15. data/lib/qless/middleware/memory_usage_monitor.rb +62 -0
  16. data/lib/qless/middleware/metriks.rb +45 -0
  17. data/lib/qless/middleware/redis_reconnect.rb +6 -3
  18. data/lib/qless/middleware/requeue_exceptions.rb +94 -0
  19. data/lib/qless/middleware/retry_exceptions.rb +38 -9
  20. data/lib/qless/middleware/sentry.rb +3 -7
  21. data/lib/qless/middleware/timeout.rb +64 -0
  22. data/lib/qless/queue.rb +90 -55
  23. data/lib/qless/server.rb +177 -130
  24. data/lib/qless/server/views/_job.erb +33 -15
  25. data/lib/qless/server/views/completed.erb +11 -0
  26. data/lib/qless/server/views/layout.erb +70 -11
  27. data/lib/qless/server/views/overview.erb +93 -53
  28. data/lib/qless/server/views/queue.erb +9 -8
  29. data/lib/qless/server/views/queues.erb +18 -1
  30. data/lib/qless/subscriber.rb +37 -22
  31. data/lib/qless/tasks.rb +5 -10
  32. data/lib/qless/test_helpers/worker_helpers.rb +55 -0
  33. data/lib/qless/version.rb +3 -1
  34. data/lib/qless/worker.rb +4 -413
  35. data/lib/qless/worker/base.rb +247 -0
  36. data/lib/qless/worker/forking.rb +245 -0
  37. data/lib/qless/worker/serial.rb +41 -0
  38. metadata +135 -52
  39. data/lib/qless/qless-core/cancel.lua +0 -101
  40. data/lib/qless/qless-core/complete.lua +0 -233
  41. data/lib/qless/qless-core/config.lua +0 -56
  42. data/lib/qless/qless-core/depends.lua +0 -65
  43. data/lib/qless/qless-core/deregister_workers.lua +0 -12
  44. data/lib/qless/qless-core/fail.lua +0 -117
  45. data/lib/qless/qless-core/failed.lua +0 -83
  46. data/lib/qless/qless-core/get.lua +0 -37
  47. data/lib/qless/qless-core/heartbeat.lua +0 -51
  48. data/lib/qless/qless-core/jobs.lua +0 -41
  49. data/lib/qless/qless-core/pause.lua +0 -18
  50. data/lib/qless/qless-core/peek.lua +0 -165
  51. data/lib/qless/qless-core/pop.lua +0 -314
  52. data/lib/qless/qless-core/priority.lua +0 -32
  53. data/lib/qless/qless-core/put.lua +0 -169
  54. data/lib/qless/qless-core/qless-lib.lua +0 -2354
  55. data/lib/qless/qless-core/qless.lua +0 -1862
  56. data/lib/qless/qless-core/queues.lua +0 -58
  57. data/lib/qless/qless-core/recur.lua +0 -190
  58. data/lib/qless/qless-core/retry.lua +0 -73
  59. data/lib/qless/qless-core/stats.lua +0 -92
  60. data/lib/qless/qless-core/tag.lua +0 -100
  61. data/lib/qless/qless-core/track.lua +0 -79
  62. data/lib/qless/qless-core/unfail.lua +0 -54
  63. data/lib/qless/qless-core/unpause.lua +0 -12
  64. data/lib/qless/qless-core/workers.lua +0 -69
  65. data/lib/qless/wait_until.rb +0 -19
@@ -1,8 +1,13 @@
1
+ # Encoding: utf-8
2
+
1
3
  require 'digest/sha1'
2
4
 
3
5
  module Qless
6
+ LuaScriptError = Class.new(Qless::Error)
7
+
8
+ # Wraps a lua script. Knows how to reload it if necessary
4
9
  class LuaScript
5
- LUA_SCRIPT_DIR = File.expand_path("../qless-core/", __FILE__)
10
+ SCRIPT_ROOT = File.expand_path('../lua', __FILE__)
6
11
 
7
12
  def initialize(name, redis)
8
13
  @name = name
@@ -12,31 +17,77 @@ module Qless
12
17
 
13
18
  attr_reader :name, :redis, :sha
14
19
 
15
- def reload()
20
+ def reload
16
21
  @sha = @redis.script(:load, script_contents)
17
22
  end
18
23
 
19
- def call(keys, argv)
20
- _call(keys, argv)
21
- rescue
22
- reload
23
- _call(keys, argv)
24
+ def call(*argv)
25
+ handle_no_script_error do
26
+ _call(*argv)
27
+ end
28
+ rescue Redis::CommandError => err
29
+ if match = err.message.match('user_script:\d+:\s*(\w+.+$)')
30
+ raise LuaScriptError.new(match[1])
31
+ else
32
+ raise err
33
+ end
24
34
  end
25
35
 
26
36
  private
27
37
 
28
38
  if USING_LEGACY_REDIS_VERSION
29
- def _call(keys, argv)
30
- @redis.evalsha(@sha, keys.length, *(keys + argv))
39
+ def _call(*argv)
40
+ @redis.evalsha(@sha, 0, *argv)
31
41
  end
32
42
  else
33
- def _call(keys, argv)
34
- @redis.evalsha(@sha, keys: keys, argv: argv)
43
+ def _call(*argv)
44
+ @redis.evalsha(@sha, keys: [], argv: argv)
45
+ end
46
+ end
47
+
48
+ def handle_no_script_error
49
+ yield
50
+ rescue ScriptNotLoadedRedisCommandError
51
+ reload
52
+ yield
53
+ end
54
+
55
+ # Module for notifying when a script hasn't yet been loaded
56
+ module ScriptNotLoadedRedisCommandError
57
+ MESSAGE = 'NOSCRIPT No matching script. Please use EVAL.'
58
+
59
+ def self.===(error)
60
+ error.is_a?(Redis::CommandError) && error.message == MESSAGE
35
61
  end
36
62
  end
37
63
 
38
64
  def script_contents
39
- @script_contents ||= File.read(File.join(LUA_SCRIPT_DIR, "#{@name}.lua"))
65
+ @script_contents ||= File.read(File.join(SCRIPT_ROOT, "#{@name}.lua"))
40
66
  end
41
67
  end
68
+
69
+ # Provides a simple way to load and use lua-based Qless plugins.
70
+ # This combines the qless-lib.lua script plus your custom script
71
+ # contents all into one script, so that your script can use
72
+ # Qless's lua API.
73
+ class LuaPlugin < LuaScript
74
+ def initialize(name, redis, plugin_contents)
75
+ @name = name
76
+ @redis = redis
77
+ @plugin_contents = plugin_contents.gsub(COMMENT_LINES_RE, '')
78
+ super(name, redis)
79
+ end
80
+
81
+ private
82
+
83
+ def script_contents
84
+ @script_contents ||= [QLESS_LIB_CONTENTS, @plugin_contents].join("\n\n")
85
+ end
86
+
87
+ COMMENT_LINES_RE = /^\s*--.*$\n?/
88
+
89
+ QLESS_LIB_CONTENTS = File.read(
90
+ File.join(SCRIPT_ROOT, 'qless-lib.lua')
91
+ ).gsub(COMMENT_LINES_RE, '')
92
+ end
42
93
  end
@@ -0,0 +1,62 @@
1
+
2
+ module Qless
3
+ module Middleware
4
+ # Monitors the memory usage of the Qless worker, instructing
5
+ # it to shutdown when memory exceeds the given :max_memory threshold.
6
+ class MemoryUsageMonitor < Module
7
+ def initialize(options)
8
+ max_memory = options.fetch(:max_memory)
9
+
10
+ module_eval do
11
+ job_counter = 0
12
+
13
+ define_method :around_perform do |job|
14
+ job_counter += 1
15
+
16
+ begin
17
+ super(job)
18
+ ensure
19
+ current_mem = MemoryUsageMonitor.current_usage_in_kb
20
+ if current_mem > max_memory
21
+ log(:info, "Exiting after job #{job_counter} since current memory " \
22
+ "(#{current_mem} KB) has exceeded max allowed memory " \
23
+ "(#{max_memory} KB).")
24
+ shutdown
25
+ end
26
+ end
27
+ end
28
+ end
29
+ end
30
+
31
+ SHELL_OUT_FOR_MEMORY = -> do
32
+ # Taken from:
33
+ # http://stackoverflow.com/a/4133642/29262
34
+ Integer(`ps -o rss= -p #{Process.pid}`)
35
+ end unless defined?(SHELL_OUT_FOR_MEMORY)
36
+
37
+ begin
38
+ require 'rusage'
39
+ rescue LoadError
40
+ warn "Could not load `rusage` gem. Falling back to shelling out "
41
+ "to get process memory usage, which is several orders of magnitude slower."
42
+
43
+ define_singleton_method(:current_usage_in_kb, &SHELL_OUT_FOR_MEMORY)
44
+ else
45
+ memory_ratio = Process.rusage.maxrss / SHELL_OUT_FOR_MEMORY.().to_f
46
+
47
+ if (800...1200).cover?(memory_ratio)
48
+ # OS X tends to return maxrss in Bytes.
49
+ def self.current_usage_in_kb
50
+ Process.rusage.maxrss / 1024
51
+ end
52
+ else
53
+ # Linux tends to return maxrss in KB.
54
+ def self.current_usage_in_kb
55
+ Process.rusage.maxrss
56
+ end
57
+ end
58
+ end
59
+ end
60
+ end
61
+ end
62
+
@@ -0,0 +1,45 @@
1
+ # Encoding: utf-8
2
+
3
+ require 'metriks'
4
+
5
+ module Qless
6
+ module Middleware
7
+ module Metriks
8
+
9
+ # Tracks the time jobs take, grouping the timings by the job class.
10
+ module TimeJobsByClass
11
+ def around_perform(job)
12
+ ::Metriks.timer("qless.job-times.#{job.klass_name}").time do
13
+ super
14
+ end
15
+ end
16
+ end
17
+
18
+ # Increments a counter each time an instance of a particular job class
19
+ # completes.
20
+ #
21
+ # Usage:
22
+ #
23
+ # Qless::Worker.class_eval do
24
+ # include Qless::Middleware::CountEvents.new(
25
+ # SomeJobClass => "event_name",
26
+ # SomeOtherJobClass => "some_other_event"
27
+ # )
28
+ # end
29
+ class CountEvents < Module
30
+ def initialize(class_to_event_map)
31
+ module_eval do # eval the block within the module instance
32
+ define_method :around_perform do |job|
33
+ super(job)
34
+ return unless job.state == 'complete'
35
+ return unless event_name = class_to_event_map[job.klass]
36
+
37
+ counter = ::Metriks.counter("qless.job-events.#{event_name}")
38
+ counter.increment
39
+ end
40
+ end
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
@@ -1,13 +1,17 @@
1
+ # Encoding: utf-8
2
+
1
3
  module Qless
2
4
  module Middleware
5
+ # A module for reconnecting to redis for each job
3
6
  module RedisReconnect
4
7
  def self.new(*redis_connections, &block)
5
8
  Module.new do
6
9
  define_singleton_method :to_s do
7
- "Qless::Middleware::RedisReconnect"
10
+ 'Qless::Middleware::RedisReconnect'
8
11
  end
12
+ define_singleton_method(:inspect, method(:to_s))
9
13
 
10
- block ||= lambda { |job| redis_connections }
14
+ block ||= ->(job) { redis_connections }
11
15
 
12
16
  define_method :around_perform do |job|
13
17
  Array(block.call(job)).each do |redis|
@@ -21,4 +25,3 @@ module Qless
21
25
  end
22
26
  end
23
27
  end
24
-
@@ -0,0 +1,94 @@
1
+ # Encoding: utf-8
2
+
3
+ module Qless
4
+ module Middleware
5
+ # This middleware is like RetryExceptions, but it doesn't use qless-core's
6
+ # internal retry/retry-tracking mechanism. Instead, it re-queues the job
7
+ # when it fails with a matched error, and increments a counter in the job's
8
+ # data.
9
+ #
10
+ # This is useful for exceptions for which you want a different
11
+ # backoff/retry strategy. The internal retry mechanism doesn't allow for
12
+ # separate tracking by exception type, and thus doesn't allow you to retry
13
+ # different exceptions a different number of times.
14
+ #
15
+ # This is particularly useful for handling resource throttling errors,
16
+ # where you may not want exponential backoff, and you may want the error
17
+ # to be retried many times, w/o having other transient errors retried so
18
+ # many times.
19
+ module RequeueExceptions
20
+ RequeueableException = Struct.new(:klass, :delay_min, :delay_span, :max_attempts) do
21
+ def self.from_splat_and_options(*klasses, options)
22
+ delay_range = options.fetch(:delay_range)
23
+ delay_min = Float(delay_range.min)
24
+ delay_span = Float(delay_range.max) - Float(delay_range.min)
25
+ max_attempts = options.fetch(:max_attempts)
26
+ klasses.map do |klass|
27
+ new(klass, delay_min, delay_span, max_attempts)
28
+ end
29
+ end
30
+
31
+ def delay
32
+ delay_min + Random.rand(delay_span)
33
+ end
34
+
35
+ def raise_if_exhausted_requeues(error, requeues)
36
+ raise error if requeues >= max_attempts
37
+ end
38
+ end
39
+
40
+ def requeue_on(*exceptions, options)
41
+ RequeueableException.from_splat_and_options(
42
+ *exceptions, options).each do |exc|
43
+ requeueable_exceptions[exc.klass] = exc
44
+ end
45
+ end
46
+
47
+ DEFAULT_ON_REQUEUE_CALLBACK = lambda { |error, job| }
48
+ def use_on_requeue_callback(&block)
49
+ @on_requeue_callback = block if block
50
+ end
51
+
52
+ def on_requeue_callback
53
+ @on_requeue_callback ||= DEFAULT_ON_REQUEUE_CALLBACK
54
+ end
55
+
56
+ def handle_exception(job, error)
57
+ config = requeuable_exception_for(error)
58
+
59
+ requeues_by_exception = (job.data['requeues_by_exception'] ||= {})
60
+ requeues_by_exception[config.klass.name] ||= 0
61
+
62
+ config.raise_if_exhausted_requeues(
63
+ error, requeues_by_exception[config.klass.name])
64
+
65
+ requeues_by_exception[config.klass.name] += 1
66
+ job.requeue(job.queue_name, delay: config.delay, data: job.data)
67
+
68
+ on_requeue_callback.call(error, job)
69
+ end
70
+
71
+ def around_perform(job)
72
+ super
73
+ rescue *requeueable_exceptions.keys => e
74
+ handle_exception(job, e)
75
+ end
76
+
77
+ def requeueable?(exception)
78
+ requeueable_exceptions.member?(exception)
79
+ end
80
+
81
+ def requeueable_exceptions
82
+ @requeueable_exceptions ||= {}
83
+ end
84
+
85
+ def requeuable_exception_for(e)
86
+ requeueable_exceptions.fetch(e.class) do
87
+ requeueable_exceptions.each do |klass, exc|
88
+ break exc if klass === e
89
+ end
90
+ end
91
+ end
92
+ end
93
+ end
94
+ end
@@ -1,13 +1,24 @@
1
+ # Encoding: utf-8
2
+
1
3
  module Qless
2
4
  module Middleware
5
+ # Auto-retries particular errors using qless-core's internal retry tracking
6
+ # mechanism. Supports a backoff strategy (typically exponential).
7
+ #
8
+ # Note: this does not support varying the number of allowed retries by
9
+ # exception type. If you want that kind of flexibility, use the
10
+ # RequeueExceptions middleware instead.
3
11
  module RetryExceptions
4
12
  def around_perform(job)
5
13
  super
6
- rescue *retryable_exception_classes => e
14
+ rescue *retryable_exception_classes => error
7
15
  raise if job.retries_left <= 0
8
16
 
9
17
  attempt_num = (job.original_retries - job.retries_left) + 1
10
- job.retry(backoff_strategy.call(attempt_num))
18
+ failure = Qless.failure_formatter.format(job, error)
19
+ job.retry(backoff_strategy.call(attempt_num, error), *failure)
20
+
21
+ on_retry_callback.call(error, job)
11
22
  end
12
23
 
13
24
  def retryable_exception_classes
@@ -18,7 +29,7 @@ module Qless
18
29
  retryable_exception_classes.push(*exception_classes)
19
30
  end
20
31
 
21
- NO_BACKOFF_STRATEGY = lambda { |num| 0 }
32
+ NO_BACKOFF_STRATEGY = ->(_num, _error) { 0 }
22
33
 
23
34
  def use_backoff_strategy(strategy = nil, &block)
24
35
  @backoff_strategy = strategy || block
@@ -28,16 +39,34 @@ module Qless
28
39
  @backoff_strategy ||= NO_BACKOFF_STRATEGY
29
40
  end
30
41
 
31
- def exponential(base, options = {})
32
- rand_fuzz = options.fetch(:rand_fuzz, 1)
33
- lambda do |num|
34
- base ** num + rand(rand_fuzz)
42
+ DEFAULT_ON_RETRY_CALLBACK = lambda { |error, job| }
43
+ def use_on_retry_callback(&block)
44
+ @on_retry_callback = block if block
45
+ end
46
+
47
+ def on_retry_callback
48
+ @on_retry_callback ||= DEFAULT_ON_RETRY_CALLBACK
49
+ end
50
+
51
+ # If `factor` is omitted it is set to `delay_seconds` to reproduce legacy
52
+ # behavior.
53
+ def exponential(delay_seconds, options={})
54
+ factor = options.fetch(:factor, delay_seconds)
55
+ fuzz_factor = options.fetch(:fuzz_factor, 0)
56
+
57
+ lambda do |retry_no, error|
58
+ unfuzzed = delay_seconds * factor**(retry_no - 1)
59
+ return unfuzzed if fuzz_factor.zero?
60
+ r = 2 * rand - 1
61
+ # r is uniformly distributed in range [-1, 1]
62
+ unfuzzed * (1 + fuzz_factor * r)
35
63
  end
36
64
  end
37
65
  end
38
66
  end
39
67
 
40
68
  # For backwards compatibility
41
- RetryExceptions = Middleware::RetryExceptions
69
+ module RetryExceptions
70
+ include Middleware::RetryExceptions
71
+ end
42
72
  end
43
-
@@ -1,3 +1,5 @@
1
+ # Encoding: utf-8
2
+
1
3
  require 'raven'
2
4
 
3
5
  module Qless
@@ -37,7 +39,6 @@ module Qless
37
39
  # Qless Web UI.
38
40
  end
39
41
 
40
-
41
42
  def job_metadata
42
43
  {
43
44
  jid: @job.jid,
@@ -55,11 +56,7 @@ module Qless
55
56
  def job_history
56
57
  @job.queue_history.map do |history_event|
57
58
  history_event.each_with_object({}) do |(key, value), hash|
58
- hash[key] = if value.is_a?(Time)
59
- value.iso8601
60
- else
61
- value
62
- end
59
+ hash[key] = value.is_a?(Time) ? value.iso8601 : value
63
60
  end
64
61
  end
65
62
  end
@@ -67,4 +64,3 @@ module Qless
67
64
  end
68
65
  end
69
66
  end
70
-
@@ -0,0 +1,64 @@
1
+ require 'timeout'
2
+ require 'qless/middleware/requeue_exceptions'
3
+
4
+ module Qless
5
+ # Unique error class used when a job is timed out by this middleware.
6
+ # Allows us to differentiate this timeout from others caused by `::Timeout::Erorr`
7
+ JobTimedoutError = Class.new(StandardError)
8
+ InvalidTimeoutError = Class.new(ArgumentError)
9
+
10
+ module Middleware
11
+ # Applies a hard time out. To use this middleware, instantiate it and pass a block; the block
12
+ # will be passed the job object (which has a `ttl` method for getting the job's remaining TTL),
13
+ # and the block should return the desired timeout in seconds.
14
+ # This allows you to set a hard constant time out to a particular job class
15
+ # (using something like `extend Qless::Middleware::Timeout.new { 60 * 60 }`),
16
+ # or a variable timeout based on the individual TTLs of each job
17
+ # (using something like `extend Qless::Middleware::Timeout.new { |job| job.ttl * 1.1 }`).
18
+ class Timeout < Module
19
+ def initialize(opts = {})
20
+ timeout_class = opts.fetch(:timeout_class, ::Timeout)
21
+ kernel_class = opts.fetch(:kernel_class, Kernel)
22
+ module_eval do
23
+ define_method :around_perform do |job|
24
+ timeout_seconds = yield job
25
+
26
+ return super(job) if timeout_seconds.nil?
27
+
28
+ if !timeout_seconds.is_a?(Numeric) || timeout_seconds <= 0
29
+ raise InvalidTimeoutError, "Timeout must be a positive number or nil, " \
30
+ "but was #{timeout_seconds}"
31
+ end
32
+
33
+ begin
34
+ timeout_class.timeout(timeout_seconds) { super(job) }
35
+ rescue ::Timeout::Error => e
36
+ error = JobTimedoutError.new("Qless: job timeout (#{timeout_seconds}) exceeded.")
37
+ error.set_backtrace(e.backtrace)
38
+ # The stalled connection to redis might be the cause of the timeout. We cannot rely
39
+ # on state of connection either (e.g., we might be in the middle of Redis call when
40
+ # timeout happend). To play it safe, we reconnect.
41
+ job.reconnect_to_redis
42
+ job.fail(*Qless.failure_formatter.format(job, error, []))
43
+ # Since we are leaving with bang (exit!), normal requeue logic does not work.
44
+ # Do it manually right here.
45
+ if self.is_a?(::Qless::Middleware::RequeueExceptions) &&
46
+ self.requeueable?(JobTimedoutError)
47
+ self.handle_exception(job, error)
48
+ end
49
+
50
+ # ::Timeout.timeout is dangerous to use as it can leave things in an inconsistent
51
+ # state. With Redis, for example, we've seen the socket buffer left with unread bytes
52
+ # on it, which can affect later redis calls. Thus, it's much safer just to exit, and
53
+ # allow the parent process to restart the worker in a known, clean state.
54
+ #
55
+ # We use 73 as a unique exit status for this case. 73 looks
56
+ # a bit like TE (Timeout::Error)
57
+ kernel_class.exit!(73)
58
+ end
59
+ end
60
+ end
61
+ end
62
+ end
63
+ end
64
+ end