hirefire-resource 0.10.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +175 -0
- data/LICENSE +21 -66
- data/README.md +34 -78
- data/hirefire-resource.gemspec +21 -15
- data/lib/hirefire/configuration.rb +31 -0
- data/lib/hirefire/errors/job_queue_latency_unsupported.rb +12 -0
- data/lib/hirefire/errors.rb +9 -0
- data/lib/hirefire/hirefire.rb +15 -0
- data/lib/hirefire/macro/bunny.rb +55 -55
- data/lib/hirefire/macro/delayed_job.rb +80 -60
- data/lib/hirefire/macro/deprecated/bunny.rb +85 -0
- data/lib/hirefire/macro/deprecated/delayed_job.rb +62 -0
- data/lib/hirefire/macro/deprecated/good_job.rb +32 -0
- data/lib/hirefire/macro/deprecated/que.rb +76 -0
- data/lib/hirefire/macro/deprecated/queue_classic.rb +28 -0
- data/lib/hirefire/macro/deprecated/resque.rb +47 -0
- data/lib/hirefire/macro/deprecated/sidekiq.rb +138 -0
- data/lib/hirefire/macro/good_job.rb +48 -13
- data/lib/hirefire/macro/que.rb +56 -36
- data/lib/hirefire/macro/queue_classic.rb +86 -0
- data/lib/hirefire/macro/resque.rb +112 -24
- data/lib/hirefire/macro/sidekiq.rb +324 -74
- data/lib/hirefire/macro/solid_queue.rb +166 -0
- data/lib/hirefire/middleware.rb +50 -103
- data/lib/hirefire/railtie.rb +1 -1
- data/lib/hirefire/resource.rb +1 -47
- data/lib/hirefire/utility.rb +22 -0
- data/lib/hirefire/version.rb +5 -0
- data/lib/hirefire/web.rb +151 -0
- data/lib/hirefire/worker.rb +39 -0
- data/lib/hirefire-resource.rb +4 -9
- metadata +50 -24
- data/.github/workflows/main.yml +0 -16
- data/.gitignore +0 -5
- data/Gemfile +0 -10
- data/Gemfile.lock +0 -49
- data/Rakefile +0 -4
- data/lib/hirefire/macro/qc.rb +0 -23
- data/lib/hirefire/macro/qu.rb +0 -27
data/lib/hirefire/macro/bunny.rb
CHANGED
|
@@ -1,78 +1,78 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "deprecated/bunny"
|
|
4
|
+
|
|
3
5
|
module HireFire
|
|
4
6
|
module Macro
|
|
5
7
|
module Bunny
|
|
8
|
+
extend HireFire::Macro::Deprecated::Bunny
|
|
9
|
+
extend HireFire::Errors::JobQueueLatencyUnsupported
|
|
10
|
+
extend HireFire::Utility
|
|
6
11
|
extend self
|
|
7
12
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
# # all queues using existing RabbitMQ connection.
|
|
13
|
-
# HireFire::Macro::Bunny.queue("queue1", "queue2", :connection => connection)
|
|
14
|
-
#
|
|
15
|
-
# # all queues using new RabbitMQ connection.
|
|
16
|
-
# HireFire::Macro::Bunny.queue("queue1", "queue2", :amqp_url => url)
|
|
17
|
-
#
|
|
18
|
-
# # all non-durable queues using new RabbitMQ connection.
|
|
19
|
-
# HireFire::Macro::Bunny.queue("queue1", "queue2", :amqp_url => url, :durable => false)
|
|
13
|
+
class ConnectionError < StandardError; end
|
|
14
|
+
|
|
15
|
+
# Calculates the total job queue size using Bunny.
|
|
20
16
|
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
17
|
+
# If an `amqp_url` is not provided, the method attempts to establish a connection using a
|
|
18
|
+
# hierarchy of environment variables for the RabbitMQ URL. It checks the following environment
|
|
19
|
+
# variables in order: `AMQP_URL`, `RABBITMQ_URL`, `RABBITMQ_BIGWIG_URL`, `CLOUDAMQP_URL`. If
|
|
20
|
+
# none of these variables are set, it defaults to a local RabbitMQ instance at
|
|
21
|
+
# "amqp://guest:guest@localhost:5672".
|
|
24
22
|
#
|
|
25
|
-
|
|
23
|
+
# @note It's important to separate jobs scheduled for future execution into a different queue
|
|
24
|
+
# from the regular queue. This is because including them in the regular queue can interfere
|
|
25
|
+
# with the accurate counting of jobs that are currently scheduled to run, leading to
|
|
26
|
+
# premature upscaling. If you want to be able to schedule jobs to run in the future,
|
|
27
|
+
# consider using the Delayed Message Plugin for RabbitMQ.
|
|
28
|
+
# @param queues [Array<String, Symbol>] Names of the queues for size measurement.
|
|
29
|
+
# @param amqp_url [String, nil] (optional) RabbitMQ URL for establishing a new connection.
|
|
30
|
+
# @return [Integer] Total job queue size.
|
|
31
|
+
# @raise [HireFire::Errors::MissingQueueError] If no queue names are specified.
|
|
32
|
+
# @example Retrieve job queue size for the "default" queue
|
|
33
|
+
# HireFire::Macro::Bunny.job_queue_size(:default)
|
|
34
|
+
# @example Retrieve job queue size across "default" and "mailer" queues
|
|
35
|
+
# HireFire::Macro::Bunny.job_queue_size(:default, :mailer)
|
|
36
|
+
# @example Use a new connection on each call using a AMQP URL
|
|
37
|
+
# HireFire::Macro::Bunny.job_queue_size(:default, amqp_url: url)
|
|
38
|
+
def job_queue_size(*queues, amqp_url: nil)
|
|
26
39
|
require "bunny"
|
|
27
40
|
|
|
28
|
-
queues
|
|
41
|
+
queues = normalize_queues(queues, allow_empty: false)
|
|
42
|
+
channel, connection = setup_channel(amqp_url)
|
|
29
43
|
|
|
30
|
-
|
|
31
|
-
queues.
|
|
32
|
-
|
|
33
|
-
|
|
44
|
+
begin
|
|
45
|
+
queues.sum { |name| channel.queue(name, passive: true).message_count }
|
|
46
|
+
ensure
|
|
47
|
+
channel&.close
|
|
48
|
+
connection&.close
|
|
34
49
|
end
|
|
50
|
+
end
|
|
35
51
|
|
|
36
|
-
|
|
37
|
-
options[:durable] = true
|
|
38
|
-
end
|
|
52
|
+
private
|
|
39
53
|
|
|
40
|
-
|
|
41
|
-
|
|
54
|
+
def setup_channel(amqp_url)
|
|
55
|
+
connection = acquire_connection(amqp_url)
|
|
42
56
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
channel = connection.create_channel
|
|
46
|
-
count_messages(channel, queues, options)
|
|
47
|
-
ensure
|
|
48
|
-
channel&.close
|
|
49
|
-
end
|
|
50
|
-
elsif options[:amqp_url]
|
|
51
|
-
connection = ::Bunny.new(options[:amqp_url])
|
|
52
|
-
begin
|
|
53
|
-
connection.start
|
|
54
|
-
channel = connection.create_channel
|
|
55
|
-
count_messages(channel, queues, options)
|
|
56
|
-
ensure
|
|
57
|
-
channel&.close
|
|
58
|
-
connection.close
|
|
59
|
-
end
|
|
57
|
+
if connection
|
|
58
|
+
[connection.create_channel, connection]
|
|
60
59
|
else
|
|
61
|
-
raise
|
|
62
|
-
|
|
60
|
+
raise ConnectionError, <<~ERROR_MSG
|
|
61
|
+
Unable to establish connection with RabbitMQ.
|
|
62
|
+
Ensure that a valid AMQP URL is provided.
|
|
63
|
+
ERROR_MSG
|
|
63
64
|
end
|
|
64
65
|
end
|
|
65
66
|
|
|
66
|
-
def
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
end
|
|
67
|
+
def acquire_connection(amqp_url)
|
|
68
|
+
url = amqp_url ||
|
|
69
|
+
ENV["AMQP_URL"] ||
|
|
70
|
+
ENV["RABBITMQ_URL"] ||
|
|
71
|
+
ENV["RABBITMQ_BIGWIG_URL"] ||
|
|
72
|
+
ENV["CLOUDAMQP_URL"] ||
|
|
73
|
+
"amqp://guest:guest@localhost:5672"
|
|
74
|
+
|
|
75
|
+
::Bunny.new(url).tap(&:start)
|
|
76
76
|
end
|
|
77
77
|
end
|
|
78
78
|
end
|
|
@@ -1,78 +1,98 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require_relative "deprecated/delayed_job"
|
|
4
|
+
|
|
3
5
|
module HireFire
|
|
4
6
|
module Macro
|
|
5
7
|
module Delayed
|
|
6
8
|
module Job
|
|
9
|
+
extend HireFire::Macro::Deprecated::Delayed::Job
|
|
10
|
+
extend HireFire::Utility
|
|
7
11
|
extend self
|
|
8
12
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# HireFire::Macro::Delayed::Job.queue(:mapper => :active_record)
|
|
15
|
-
#
|
|
16
|
-
# # all queues using ActiveRecord <= 2.3.x mapper.
|
|
17
|
-
# HireFire::Macro::Delayed::Job.queue(:mapper => :active_record_2)
|
|
18
|
-
#
|
|
19
|
-
# # only "email" queue with Mongoid mapper.
|
|
20
|
-
# HireFire::Macro::Delayed::Job.queue("email", :mapper => :mongoid)
|
|
21
|
-
#
|
|
22
|
-
# # "audio" and "video" queues with ActiveRecord mapper.
|
|
23
|
-
# HireFire::Macro::Delayed::Job.queue("audio", "video", :mapper => :active_record)
|
|
24
|
-
#
|
|
25
|
-
# # all queues with a maximum priority of 20
|
|
26
|
-
# HireFire::Macro::Delayed::Job.queue(:max_priority => 20, :mapper => :active_record)
|
|
27
|
-
#
|
|
28
|
-
# # all queues with a minimum priority of 5
|
|
29
|
-
# HireFire::Macro::Delayed::Job.queue(:min_priority => 5, :mapper => :active_record)
|
|
30
|
-
#
|
|
31
|
-
# @param [Array] queues provide one or more queue names, or none for "all".
|
|
32
|
-
# Last argument can pass in a Hash containing :mapper => :active_record or :mapper => :mongoid
|
|
33
|
-
# @return [Integer] the number of jobs in the queue(s).
|
|
13
|
+
class MapperNotDetectedError < StandardError; end
|
|
14
|
+
|
|
15
|
+
# Calculates the maximum job queue latency using Delayed::Job. If no queues are specified,
|
|
16
|
+
# it measures latency across all available queues. This method supports both ActiveRecord
|
|
17
|
+
# and Mongoid mappers.
|
|
34
18
|
#
|
|
35
|
-
|
|
36
|
-
|
|
19
|
+
# @param queues [Array<String, Symbol>] (optional) Names of the queues for latency
|
|
20
|
+
# measurement. If not provided, latency is measured across all queues.
|
|
21
|
+
# @return [Float] Maximum job queue latency in seconds.
|
|
22
|
+
# @example Calculate latency across all queues
|
|
23
|
+
# HireFire::Macro::Delayed::Job.job_queue_latency
|
|
24
|
+
# @example Calculate latency for the "default" queue
|
|
25
|
+
# HireFire::Macro::Delayed::Job.job_queue_latency(:default)
|
|
26
|
+
# @example Calculate latency across "default" and "mailer" queues
|
|
27
|
+
# HireFire::Macro::Delayed::Job.job_queue_latency(:default, :mailer)
|
|
28
|
+
def job_queue_latency(*queues)
|
|
29
|
+
queues = normalize_queues(queues, allow_empty: true)
|
|
37
30
|
|
|
38
|
-
|
|
39
|
-
|
|
31
|
+
query =
|
|
32
|
+
::Delayed::Job
|
|
33
|
+
.where(run_at: ..Time.now)
|
|
34
|
+
.where(failed_at: nil)
|
|
35
|
+
.order(run_at: :asc)
|
|
36
|
+
|
|
37
|
+
if queues.any?
|
|
38
|
+
case mapper
|
|
39
|
+
when :active_record
|
|
40
|
+
query = query.where(queue: queues)
|
|
41
|
+
when :mongoid
|
|
42
|
+
query = query.in(queue: queues.to_a)
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
if (job = query.first)
|
|
47
|
+
Time.now - job.run_at
|
|
40
48
|
else
|
|
41
|
-
|
|
49
|
+
0.0
|
|
42
50
|
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Calculates the total job queue size using Delayed::Job. If no queues are specified, it
|
|
54
|
+
# measures size across all available queues. This method supports both ActiveRecord and
|
|
55
|
+
# Mongoid mappers.
|
|
56
|
+
#
|
|
57
|
+
# @param queues [Array<String, Symbol>] (optional) Names of the queues for size measurement.
|
|
58
|
+
# If not provided, size is measured across all queues.
|
|
59
|
+
# @return [Integer] Total job queue size.
|
|
60
|
+
# @example Calculate size across all queues
|
|
61
|
+
# HireFire::Macro::Delayed::Job.job_queue_size
|
|
62
|
+
# @example Calculate size of the "default" queue
|
|
63
|
+
# HireFire::Macro::Delayed::Job.job_queue_size(:default)
|
|
64
|
+
# @example Calculate size across "default" and "mailer" queues
|
|
65
|
+
# HireFire::Macro::Delayed::Job.job_queue_size(:default, :mailer)
|
|
66
|
+
def job_queue_size(*queues)
|
|
67
|
+
queues = normalize_queues(queues, allow_empty: true)
|
|
68
|
+
|
|
69
|
+
query =
|
|
70
|
+
::Delayed::Job
|
|
71
|
+
.where(run_at: ..Time.now)
|
|
72
|
+
.where(failed_at: nil)
|
|
43
73
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
c = c.where("priority <= ?", options[:max_priority]) if options.key?(:max_priority)
|
|
51
|
-
c = c.where(queue: queues) unless queues.empty?
|
|
52
|
-
c.count.tap { ActiveRecord::Base.clear_active_connections! }
|
|
53
|
-
when :active_record_2
|
|
54
|
-
c = ::Delayed::Job
|
|
55
|
-
c = c.scoped(conditions: ["run_at <= ? AND failed_at is NULL", Time.now.utc])
|
|
56
|
-
c = c.scoped(conditions: ["priority >= ?", options[:min_priority]]) if options.key?(:min_priority)
|
|
57
|
-
c = c.scoped(conditions: ["priority <= ?", options[:max_priority]]) if options.key?(:max_priority)
|
|
58
|
-
# There is no queue column in delayed_job <= 2.x
|
|
59
|
-
c.count.tap do
|
|
60
|
-
if ActiveRecord::Base.respond_to?(:clear_active_connections!)
|
|
61
|
-
ActiveRecord::Base.clear_active_connections!
|
|
62
|
-
end
|
|
74
|
+
if queues.any?
|
|
75
|
+
case mapper
|
|
76
|
+
when :active_record
|
|
77
|
+
query = query.where(queue: queues)
|
|
78
|
+
when :mongoid
|
|
79
|
+
query = query.in(queue: queues.to_a)
|
|
63
80
|
end
|
|
64
|
-
when :mongoid
|
|
65
|
-
c = ::Delayed::Job
|
|
66
|
-
c = c.where(failed_at: nil)
|
|
67
|
-
c = c.where(:run_at.lte => Time.now.utc)
|
|
68
|
-
c = c.where(:priority.gte => options[:min_priority]) if options.key?(:min_priority)
|
|
69
|
-
c = c.where(:priority.lte => options[:max_priority]) if options.key?(:max_priority)
|
|
70
|
-
c = c.where(:queue.in => queues) unless queues.empty?
|
|
71
|
-
c.count
|
|
72
|
-
else
|
|
73
|
-
raise %(Must pass in :mapper => :active_record or :mapper => :mongoid\n) +
|
|
74
|
-
%{For example: HireFire::Macro::Delayed::Job.queue("worker", :mapper => :active_record)}
|
|
75
81
|
end
|
|
82
|
+
|
|
83
|
+
query.count
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
private
|
|
87
|
+
|
|
88
|
+
def mapper
|
|
89
|
+
return :active_record if defined?(::ActiveRecord::Base) &&
|
|
90
|
+
::Delayed::Job.ancestors.include?(::ActiveRecord::Base)
|
|
91
|
+
|
|
92
|
+
return :mongoid if defined?(::Mongoid::Document) &&
|
|
93
|
+
::Delayed::Job.ancestors.include?(::Mongoid::Document)
|
|
94
|
+
|
|
95
|
+
raise MapperNotDetectedError, "Unable to detect the appropriate mapper."
|
|
76
96
|
end
|
|
77
97
|
end
|
|
78
98
|
end
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
# Provides backward compatibility with the deprecated Bunny macro.
|
|
7
|
+
# For new implementations, refer to {HireFire::Macro::Bunny}.
|
|
8
|
+
module Bunny
|
|
9
|
+
# Retrieves the total number of jobs in the specified queue(s).
|
|
10
|
+
#
|
|
11
|
+
# This method allows querying multiple queues and supports both existing and new RabbitMQ
|
|
12
|
+
# connections. By default, queues are considered durable unless specified otherwise.
|
|
13
|
+
#
|
|
14
|
+
# @param queues [Array<String, Symbol>] Queue names to query.
|
|
15
|
+
# The last argument can be a Hash with either :connection or :amqp_url.
|
|
16
|
+
# @option queues [Bunny::Session, nil] :connection An existing RabbitMQ connection.
|
|
17
|
+
# @option queues [String, nil] :amqp_url RabbitMQ URL for initializing a new connection.
|
|
18
|
+
# @option queues [Boolean] :durable (true) Set to false for non-durable queues.
|
|
19
|
+
# @option queues [Integer, nil] :"x-max-priority" (nil) The maximum priority level for the queue.
|
|
20
|
+
# If specified, it overrides the default priority settings for the queue.
|
|
21
|
+
# @return [Integer] Total number of jobs in the specified queues.
|
|
22
|
+
# @raise [ArgumentError] Raises an error if neither :connection nor :amqp_url are provided.
|
|
23
|
+
# @example Querying the default queue using an existing RabbitMQ connection
|
|
24
|
+
# HireFire::Macro::Bunny.queue("default", connection: connection)
|
|
25
|
+
# @example Querying the default queue using a new RabbitMQ connection
|
|
26
|
+
# HireFire::Macro::Bunny.queue("default", amqp_url: url)
|
|
27
|
+
# @example Querying the "default" and "critical" non-durable queues
|
|
28
|
+
# HireFire::Macro::Bunny.queue("default", "critical", amqp_url: url, durable: false)
|
|
29
|
+
# @example Querying a priority queue
|
|
30
|
+
# HireFire::Macro::Bunny.queue("priority_queue", connection: connection, "x-max-priority": 10)
|
|
31
|
+
def queue(*queues)
|
|
32
|
+
require "bunny"
|
|
33
|
+
|
|
34
|
+
queues.flatten!
|
|
35
|
+
options = queues.last.is_a?(Hash) ? queues.pop : {}
|
|
36
|
+
options[:durable] = true if options[:durable].nil?
|
|
37
|
+
|
|
38
|
+
if options[:connection]
|
|
39
|
+
connection = options[:connection]
|
|
40
|
+
channel = nil
|
|
41
|
+
begin
|
|
42
|
+
channel = connection.create_channel
|
|
43
|
+
Private.count_messages(channel, queues, options)
|
|
44
|
+
ensure
|
|
45
|
+
channel&.close
|
|
46
|
+
end
|
|
47
|
+
elsif options[:amqp_url]
|
|
48
|
+
connection = ::Bunny.new(options[:amqp_url])
|
|
49
|
+
begin
|
|
50
|
+
connection.start
|
|
51
|
+
channel = connection.create_channel
|
|
52
|
+
Private.count_messages(channel, queues, options)
|
|
53
|
+
ensure
|
|
54
|
+
channel&.close
|
|
55
|
+
connection.close
|
|
56
|
+
end
|
|
57
|
+
else
|
|
58
|
+
raise ArgumentError, "Must pass either :connection => rabbitmq_connection or :amqp_url => url." \
|
|
59
|
+
"For example: HireFire::Macro::Bunny.queue(\"queue1\", connection: rabbitmq_connection)"
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# @!visibility private
|
|
64
|
+
module Private
|
|
65
|
+
extend self
|
|
66
|
+
|
|
67
|
+
# Counts the number of messages in the specified queues.
|
|
68
|
+
#
|
|
69
|
+
# @param channel [Bunny::Channel] The channel to interact with RabbitMQ.
|
|
70
|
+
# @param queues [Array<String, Symbol>] The names of the queues to count messages from.
|
|
71
|
+
# @param options [Hash] The options for the queues, including durability and priority settings.
|
|
72
|
+
# @return [Integer] The total number of messages across all specified queues.
|
|
73
|
+
def count_messages(channel, queues, options)
|
|
74
|
+
queues.inject(0) do |sum, queue|
|
|
75
|
+
queue_options = {durable: options[:durable]}
|
|
76
|
+
queue_options[:arguments] = {"x-max-priority" => options[:"x-max-priority"]} if options.key?(:"x-max-priority")
|
|
77
|
+
queue = channel.queue(queue.to_s, **queue_options)
|
|
78
|
+
sum + queue.message_count
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
module Delayed
|
|
7
|
+
# Provides backward compatibility with the deprecated Delayed::Job macro.
|
|
8
|
+
# For new implementations, refer to {HireFire::Macro::Delayed::Job}.
|
|
9
|
+
module Job
|
|
10
|
+
# Retrieves the total number of jobs in the specified queue(s).
|
|
11
|
+
#
|
|
12
|
+
# This method supports querying jobs across different ORMs (Object-Relational Mappings)
|
|
13
|
+
# such as ActiveRecord and Mongoid. It allows specifying queue names and priority limits.
|
|
14
|
+
#
|
|
15
|
+
# @param queues [Array<String, Symbol>] Queue names to query.
|
|
16
|
+
# The last argument can be a Hash with :mapper, :min_priority, and/or :max_priority keys.
|
|
17
|
+
# @option queues [Symbol] :mapper (:active_record, :active_record_2, :mongoid) The ORM mapper to use.
|
|
18
|
+
# @option queues [Integer, nil] :min_priority (nil) The minimum job priority to include in the count.
|
|
19
|
+
# If not specified, no lower limit is applied.
|
|
20
|
+
# @option queues [Integer, nil] :max_priority (nil) The maximum job priority to include in the count.
|
|
21
|
+
# If not specified, no upper limit is applied.
|
|
22
|
+
# @return [Integer] Total number of jobs in the specified queues.
|
|
23
|
+
# @raise [ArgumentError] Raises an error if a valid :mapper option is not provided.
|
|
24
|
+
# @example Querying all queues using ActiveRecord mapper
|
|
25
|
+
# HireFire::Macro::Delayed::Job.queue(mapper: :active_record)
|
|
26
|
+
# @example Querying specific queues with Mongoid mapper
|
|
27
|
+
# HireFire::Macro::Delayed::Job.queue("default", mapper: :mongoid)
|
|
28
|
+
# @example Query all queues scoped to a priority range
|
|
29
|
+
# HireFire::Macro::Delayed::Job.queue(max_priority: 20, min_priority: 5, mapper: :active_record)
|
|
30
|
+
def queue(*queues)
|
|
31
|
+
queues.flatten!
|
|
32
|
+
options = queues.last.is_a?(Hash) ? queues.pop : {}
|
|
33
|
+
|
|
34
|
+
case options[:mapper]
|
|
35
|
+
when :active_record
|
|
36
|
+
query = ::Delayed::Job.where(failed_at: nil, run_at: ..Time.now.utc)
|
|
37
|
+
query = query.where(priority: options[:min_priority]..) if options.key?(:min_priority)
|
|
38
|
+
query = query.where(priority: ..options[:max_priority]) if options.key?(:max_priority)
|
|
39
|
+
query = query.where(queue: queues) unless queues.empty?
|
|
40
|
+
query.count.tap { ActiveRecord::Base.clear_active_connections! }
|
|
41
|
+
when :active_record_2
|
|
42
|
+
# Note: There is no queue column in delayed_job <= 2.x
|
|
43
|
+
query = ::Delayed::Job.scoped(conditions: ["run_at <= ? AND failed_at is NULL", Time.now.utc])
|
|
44
|
+
query = query.scoped(conditions: ["priority >= ?", options[:min_priority]]) if options.key?(:min_priority)
|
|
45
|
+
query = query.scoped(conditions: ["priority <= ?", options[:max_priority]]) if options.key?(:max_priority)
|
|
46
|
+
query.count.tap { ActiveRecord::Base.clear_active_connections! if ActiveRecord::Base.respond_to?(:clear_active_connections!) }
|
|
47
|
+
when :mongoid
|
|
48
|
+
query = ::Delayed::Job.where(:failed_at => nil, :run_at.lte => Time.now.utc)
|
|
49
|
+
query = query.where(:priority.gte => options[:min_priority]) if options.key?(:min_priority)
|
|
50
|
+
query = query.where(:priority.lte => options[:max_priority]) if options.key?(:max_priority)
|
|
51
|
+
query = query.where(:queue.in => queues) unless queues.empty?
|
|
52
|
+
query.count
|
|
53
|
+
else
|
|
54
|
+
raise ArgumentError, "Must pass either :mapper => :active_record or :mapper => :mongoid. " \
|
|
55
|
+
"For example: HireFire::Macro::Delayed::Job.queue(\"worker\", mapper: :active_record)"
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
# Provides backward compatibility with the deprecated GoodJob macro.
|
|
7
|
+
# For new implementations, refer to {HireFire::Macro::GoodJob}.
|
|
8
|
+
module GoodJob
|
|
9
|
+
# Retrieves the total number of jobs in the specified queue(s) using GoodJob.
|
|
10
|
+
#
|
|
11
|
+
# This method queries the PostgreSQL database through GoodJob. It's capable
|
|
12
|
+
# of counting jobs across different queues or all queues if none specified.
|
|
13
|
+
# The method checks for the existence of ::GoodJob::Execution or ::GoodJob::Job
|
|
14
|
+
# to determine the base class to use for querying.
|
|
15
|
+
#
|
|
16
|
+
# @param queues [Array<String>] The names of the queues to count.
|
|
17
|
+
# Pass an empty array or no arguments to count jobs in all queues.
|
|
18
|
+
# @return [Integer] Total number of jobs in the specified queues.
|
|
19
|
+
# @example Counting jobs in all queues
|
|
20
|
+
# HireFire::Macro::GoodJob.queue
|
|
21
|
+
# @example Counting jobs in the "default" queue
|
|
22
|
+
# HireFire::Macro::GoodJob.queue("default")
|
|
23
|
+
def queue(*queues)
|
|
24
|
+
base_class = defined?(::GoodJob::Execution) ? ::GoodJob::Execution : ::GoodJob::Job
|
|
25
|
+
scope = base_class.only_scheduled.unfinished
|
|
26
|
+
scope = scope.where(queue_name: queues) if queues.any?
|
|
27
|
+
scope.count
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
# Provides backward compatibility with the deprecated Que macro.
|
|
7
|
+
# For new implementations, refer to {HireFire::Macro::Que}.
|
|
8
|
+
module Que
|
|
9
|
+
# Retrieves the total number of jobs in the specified queue(s) using Que.
|
|
10
|
+
#
|
|
11
|
+
# This method queries the PostgreSQL database through Que. It can count jobs
|
|
12
|
+
# in specified queues or all queues if no specific queue is provided.
|
|
13
|
+
# The method determines the base query depending on the Que version detected.
|
|
14
|
+
#
|
|
15
|
+
# @param queues [Array<String>] The names of the queues to count.
|
|
16
|
+
# Pass an empty array or no arguments to count jobs in all queues.
|
|
17
|
+
# @return [Integer] Total number of jobs in the specified queues.
|
|
18
|
+
# @example Counting jobs in all queues
|
|
19
|
+
# HireFire::Macro::Que.queue
|
|
20
|
+
# @example Counting jobs in the "default" queue
|
|
21
|
+
# HireFire::Macro::Que.queue("default")
|
|
22
|
+
def queue(*queues)
|
|
23
|
+
query = queues.empty? ? Private.base_query : "#{Private.base_query} AND queue IN (#{Private.names(queues)})"
|
|
24
|
+
results = ::Que.execute(query).first
|
|
25
|
+
(results[:total] || results["total"]).to_i
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# @!visibility private
|
|
29
|
+
module Private
|
|
30
|
+
extend self
|
|
31
|
+
|
|
32
|
+
# Determines the base query to use for counting jobs, depending on the Que version.
|
|
33
|
+
#
|
|
34
|
+
# @return [String] The base SQL query string.
|
|
35
|
+
def base_query
|
|
36
|
+
return QUE_V0_QUERY if defined?(::Que::Version)
|
|
37
|
+
return QUE_V1_QUERY if defined?(::Que::VERSION)
|
|
38
|
+
raise "Couldn't find Que version"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# Formats queue names for SQL query.
|
|
42
|
+
#
|
|
43
|
+
# @param queues [Array<String>] The names of the queues.
|
|
44
|
+
# @return [String] Formatted queue names for SQL IN clause.
|
|
45
|
+
def names(queues)
|
|
46
|
+
queues.map { |queue| "'#{queue}'" }.join(",")
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Formats and freezes a SQL query string for use.
|
|
50
|
+
#
|
|
51
|
+
# @param query [String] The raw SQL query string.
|
|
52
|
+
# @return [String] The formatted and frozen SQL query string.
|
|
53
|
+
def query_const(query)
|
|
54
|
+
query.gsub(/\s+/, " ").strip.freeze
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# SQL query string for Que version 0.
|
|
58
|
+
QUE_V0_QUERY = query_const(<<-QUERY)
|
|
59
|
+
SELECT COUNT(*) AS total
|
|
60
|
+
FROM que_jobs
|
|
61
|
+
WHERE run_at < NOW()
|
|
62
|
+
QUERY
|
|
63
|
+
|
|
64
|
+
# SQL query string for Que version 1.
|
|
65
|
+
QUE_V1_QUERY = query_const(<<-QUERY)
|
|
66
|
+
SELECT COUNT(*) AS total
|
|
67
|
+
FROM que_jobs
|
|
68
|
+
WHERE finished_at IS NULL
|
|
69
|
+
AND expired_at IS NULL
|
|
70
|
+
AND run_at <= NOW()
|
|
71
|
+
QUERY
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
# Provides backward compatibility with the deprecated QC macro.
|
|
7
|
+
# For new implementations, refer to {HireFire::Macro::QC}.
|
|
8
|
+
module QC
|
|
9
|
+
# Retrieves the total number of jobs in the specified queue using QueueClassic.
|
|
10
|
+
#
|
|
11
|
+
# This method queries the PostgreSQL database through QueueClassic. It's capable of counting
|
|
12
|
+
# jobs in a specific queue, defaulting to the "default" queue if none is specified.
|
|
13
|
+
# It utilizes the QC::Queue class to interface with the QueueClassic system.
|
|
14
|
+
#
|
|
15
|
+
# @param queue [String, Symbol] The name of the queue to count.
|
|
16
|
+
# Defaults to "default" if no queue name is provided.
|
|
17
|
+
# @return [Integer] Total number of jobs in the specified queue.
|
|
18
|
+
# @example Counting jobs in the "default" queue
|
|
19
|
+
# HireFire::Macro::QC.queue
|
|
20
|
+
# @example Counting jobs in the "critical" queue
|
|
21
|
+
# HireFire::Macro::QC.queue("critical")
|
|
22
|
+
def queue(queue = "default")
|
|
23
|
+
::QC::Queue.new(queue.to_s).count
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HireFire
|
|
4
|
+
module Macro
|
|
5
|
+
module Deprecated
|
|
6
|
+
# Provides backward compatibility with the deprecated Resque macro.
|
|
7
|
+
# For new implementations, refer to {HireFire::Macro::Resque}.
|
|
8
|
+
module Resque
|
|
9
|
+
# Retrieves the total number of jobs in the specified Resque queue(s).
|
|
10
|
+
#
|
|
11
|
+
# This method counts the number of jobs in either specific queues or all queues if none are
|
|
12
|
+
# specified. It includes both queued and in-progress jobs.
|
|
13
|
+
#
|
|
14
|
+
# @param queues [Array<String, Symbol>] Queue names to count jobs in.
|
|
15
|
+
# Pass an empty array or no arguments to count jobs in all queues.
|
|
16
|
+
# @return [Integer] Total number of jobs in the specified queues.
|
|
17
|
+
# @example Counting jobs in all queues
|
|
18
|
+
# HireFire::Macro::Resque.queue
|
|
19
|
+
# @example Counting jobs in the "default" queue
|
|
20
|
+
# HireFire::Macro::Resque.queue("default")
|
|
21
|
+
# @example Counting jobs in both "default" and "critical" queues
|
|
22
|
+
# HireFire::Macro::Resque.queue("default", "critical")
|
|
23
|
+
def queue(*queues)
|
|
24
|
+
queues = queues.flatten.map(&:to_s)
|
|
25
|
+
queues = ::Resque.queues if queues.empty?
|
|
26
|
+
|
|
27
|
+
return 0 if queues.empty?
|
|
28
|
+
|
|
29
|
+
redis = ::Resque.redis
|
|
30
|
+
worker_ids = Array(redis.smembers(:workers)).compact
|
|
31
|
+
raw_jobs = redis.pipelined do |redis|
|
|
32
|
+
worker_ids.map { |id| redis.get("worker:#{id}") }
|
|
33
|
+
end
|
|
34
|
+
jobs_in_progress = raw_jobs.map { |raw_job| ::Resque.decode(raw_job) || {} }
|
|
35
|
+
|
|
36
|
+
jobs_in_queues = redis.pipelined do |redis|
|
|
37
|
+
queues.map { |queue| redis.llen("queue:#{queue}") }
|
|
38
|
+
end.map(&:to_i).sum
|
|
39
|
+
|
|
40
|
+
in_progress_count = jobs_in_progress.count { |job| queues.include?(job["queue"]) }
|
|
41
|
+
|
|
42
|
+
jobs_in_queues + in_progress_count
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|