hirefire-resource 0.7.3 → 0.9.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +25 -1
- data/README.md +3 -2
- data/Rakefile +2 -0
- data/hirefire-resource.gemspec +7 -3
- data/lib/hirefire-resource.rb +2 -3
- data/lib/hirefire/cli.rb +1 -2
- data/lib/hirefire/macro/bunny.rb +3 -3
- data/lib/hirefire/macro/delayed_job.rb +1 -2
- data/lib/hirefire/macro/good_job.rb +25 -0
- data/lib/hirefire/macro/qc.rb +1 -2
- data/lib/hirefire/macro/qu.rb +1 -2
- data/lib/hirefire/macro/que.rb +34 -8
- data/lib/hirefire/macro/resque.rb +1 -1
- data/lib/hirefire/macro/sidekiq.rb +12 -8
- data/lib/hirefire/middleware.rb +5 -5
- data/lib/hirefire/railtie.rb +1 -1
- data/lib/hirefire/resource.rb +2 -2
- metadata +12 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f395466dc5805c8bd4d686f22181e515ac14a600e15e062a8e55d582dc6a0e57
|
4
|
+
data.tar.gz: ad6fd4856109de19979e8a975f65c0697672f09ccbbc67ef635c1e83384533d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fff67ee5f1bb27f874fbb2f401681ab1fb37c6904f62ac10ab42fb38b256ba1f8638407fc78f914e3aa40c5178c6bffff95a0a40dfab4882aa2726fc815bf8a0
|
7
|
+
data.tar.gz: 52cc6314c95f940140afb1798e2afac18c4cbc7a494f8cc6ea4e3a111aee6fa53876dcb9f85f924a02e843eb560cbc7b755f3ed7e5522d00f1b23f7727e7c70c
|
data/CHANGELOG.md
CHANGED
@@ -1,4 +1,28 @@
|
|
1
|
-
## v0.
|
1
|
+
## v0.9.0/master
|
2
|
+
|
3
|
+
* Add `skip_working` to Sidekiq macro
|
4
|
+
* Use separate queries for Que 0.x and 1.x
|
5
|
+
* Remove `# encoding: utf-8` magic comments
|
6
|
+
* Add `# frozen_string_literal: true` magic comments
|
7
|
+
|
8
|
+
## v0.8.1
|
9
|
+
|
10
|
+
* Correct GoodJob macro to not count finished jobs.
|
11
|
+
|
12
|
+
## v0.8.0
|
13
|
+
|
14
|
+
* Add GoodJob macro for `good_job` adapter. https://github.com/bensheldon/good_job
|
15
|
+
|
16
|
+
## v0.7.5
|
17
|
+
|
18
|
+
* Fix compatibility issue with Que 1.x (backwards-compatible with Que 0.x).
|
19
|
+
|
20
|
+
## v0.7.4
|
21
|
+
|
22
|
+
* Attempt to fix an issue where the STDOUT IO Stream has been closed for an unknown reason.
|
23
|
+
* This resulted in errors in an application with `log_queue_metrics` enabled after a random period of time.
|
24
|
+
|
25
|
+
## v0.7.3
|
2
26
|
|
3
27
|
* Added priority queue support for bunny message count.
|
4
28
|
* Allows for passing in the `x-max-priority` option when opening up a queue to check the messages remaining.
|
data/README.md
CHANGED
@@ -6,13 +6,14 @@ Load-based scaling, schedule-based scaling, dyno crash recovery, for [Heroku](ht
|
|
6
6
|
|
7
7
|
It supports practically any worker library. We provide out-of-the-box support for:
|
8
8
|
|
9
|
+
* Sidekiq
|
9
10
|
* Delayed Job
|
10
11
|
* Resque
|
12
|
+
* GoodJob
|
11
13
|
* Qu
|
14
|
+
* Que
|
12
15
|
* QueueClassic
|
13
|
-
* Sidekiq
|
14
16
|
* Bunny
|
15
|
-
* Que
|
16
17
|
|
17
18
|
*Note that you can write your own worker queue logic for almost any other worker library as well.
|
18
19
|
HireFire can scale multiple individual worker libraries at the same time, as well as multiple individual queues for any worker library.*
|
data/Rakefile
CHANGED
data/hirefire-resource.gemspec
CHANGED
@@ -1,8 +1,6 @@
|
|
1
|
-
# encoding: utf-8
|
2
|
-
|
3
1
|
Gem::Specification.new do |gem|
|
4
2
|
gem.name = "hirefire-resource"
|
5
|
-
gem.version = "0.
|
3
|
+
gem.version = "0.9.0"
|
6
4
|
gem.platform = Gem::Platform::RUBY
|
7
5
|
gem.authors = "Michael van Rooijen"
|
8
6
|
gem.email = "michael@hirefire.io"
|
@@ -10,6 +8,12 @@ Gem::Specification.new do |gem|
|
|
10
8
|
gem.summary = "Autoscaling for your Heroku dynos"
|
11
9
|
gem.description = "Load- and schedule-based scaling for web- and worker dynos"
|
12
10
|
gem.licenses = ["Apache License"]
|
11
|
+
gem.metadata = {
|
12
|
+
"homepage_uri" => "https://www.hirefire.io",
|
13
|
+
"changelog_uri" => "https://github.com/hirefire/hirefire-resource/blob/master/CHANGELOG.md",
|
14
|
+
"source_code_uri" => "https://github.com/hirefire/hirefire-resource/",
|
15
|
+
"bug_tracker_uri" => "https://github.com/hirefire/hirefire-resource/issues",
|
16
|
+
}
|
13
17
|
|
14
18
|
gem.files = %x[git ls-files].split("\n")
|
15
19
|
gem.executables = ["hirefire", "hirefireapp"]
|
data/lib/hirefire-resource.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
HIREFIRE_PATH = File.expand_path("../hirefire", __FILE__)
|
4
4
|
|
@@ -6,9 +6,8 @@ HIREFIRE_PATH = File.expand_path("../hirefire", __FILE__)
|
|
6
6
|
require "#{HIREFIRE_PATH}/#{file}"
|
7
7
|
end
|
8
8
|
|
9
|
-
%w[delayed_job resque sidekiq qu qc bunny que].each do |file|
|
9
|
+
%w[delayed_job resque sidekiq qu qc bunny que good_job].each do |file|
|
10
10
|
require "#{HIREFIRE_PATH}/macro/#{file}"
|
11
11
|
end
|
12
12
|
|
13
13
|
require "#{HIREFIRE_PATH}/railtie" if defined?(Rails::Railtie)
|
14
|
-
|
data/lib/hirefire/cli.rb
CHANGED
data/lib/hirefire/macro/bunny.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HireFire
|
4
4
|
module Macro
|
@@ -70,9 +70,9 @@ module HireFire
|
|
70
70
|
|
71
71
|
def count_messages(channel, queue_names, options)
|
72
72
|
queue_names.inject(0) do |sum, queue_name|
|
73
|
-
if options.key?(
|
73
|
+
if options.key?(:'x-max-priority')
|
74
74
|
queue = channel.queue(queue_name, :durable => options[:durable],
|
75
|
-
:arguments => {"x-max-priority" => options[
|
75
|
+
:arguments => {"x-max-priority" => options[:'x-max-priority']})
|
76
76
|
else
|
77
77
|
queue = channel.queue(queue_name, :durable => options[:durable])
|
78
78
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HireFire
|
4
|
+
module Macro
|
5
|
+
module GoodJob
|
6
|
+
extend self
|
7
|
+
|
8
|
+
# Queries the PostgreSQL database through GoodJob in order to
|
9
|
+
# count the amount of jobs in the specified queue.
|
10
|
+
#
|
11
|
+
# @example Queue Macro Usage
|
12
|
+
# HireFire::Macro::GoodJob.queue # counts all queues.
|
13
|
+
# HireFire::Macro::GoodJob.queue("email") # counts the `email` queue.
|
14
|
+
#
|
15
|
+
# @param [String] queue the queue name to count. (default: nil # gets all queues)
|
16
|
+
# @return [Integer] the number of jobs in the queue(s).
|
17
|
+
#
|
18
|
+
def queue(*queues)
|
19
|
+
scope = ::GoodJob::Job.only_scheduled.unfinished
|
20
|
+
scope = scope.where(queue_name: queues) if queues.any?
|
21
|
+
scope.count
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/hirefire/macro/qc.rb
CHANGED
data/lib/hirefire/macro/qu.rb
CHANGED
data/lib/hirefire/macro/que.rb
CHANGED
@@ -1,12 +1,8 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HireFire
|
4
4
|
module Macro
|
5
5
|
module Que
|
6
|
-
QUERY = %{
|
7
|
-
SELECT count(*) AS total
|
8
|
-
FROM que_jobs WHERE run_at < now() }.freeze
|
9
|
-
|
10
6
|
extend self
|
11
7
|
|
12
8
|
# Queries the PostgreSQL database through Que in order to
|
@@ -19,11 +15,41 @@ FROM que_jobs WHERE run_at < now() }.freeze
|
|
19
15
|
# @param [String] queue the queue name to count. (default: nil # gets all queues)
|
20
16
|
# @return [Integer] the number of jobs in the queue(s).
|
21
17
|
#
|
22
|
-
def queue(
|
23
|
-
query
|
18
|
+
def queue(*queues)
|
19
|
+
query = queues.empty? && base_query || base_query + " AND queue IN (#{names(queues)})"
|
24
20
|
results = ::Que.execute(query).first
|
25
|
-
results["total"].to_i
|
21
|
+
(results[:total] || results["total"]).to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
def base_query
|
27
|
+
return QUE_V0_QUERY if defined?(::Que::Version)
|
28
|
+
return QUE_V1_QUERY if defined?(::Que::VERSION)
|
29
|
+
raise "Couldn't find Que version"
|
30
|
+
end
|
31
|
+
|
32
|
+
def names(queues)
|
33
|
+
queues.map { |queue| "'#{queue}'" }.join(",")
|
34
|
+
end
|
35
|
+
|
36
|
+
def query_const(query)
|
37
|
+
query.gsub(/\s+/, " ").strip.freeze
|
26
38
|
end
|
39
|
+
|
40
|
+
QUE_V0_QUERY = query_const(<<-QUERY)
|
41
|
+
SELECT COUNT(*) AS total
|
42
|
+
FROM que_jobs
|
43
|
+
WHERE run_at < NOW()
|
44
|
+
QUERY
|
45
|
+
|
46
|
+
QUE_V1_QUERY = query_const(<<-QUERY)
|
47
|
+
SELECT COUNT(*) AS total
|
48
|
+
FROM que_jobs
|
49
|
+
WHERE finished_at IS NULL
|
50
|
+
AND expired_at IS NULL
|
51
|
+
AND run_at <= NOW()
|
52
|
+
QUERY
|
27
53
|
end
|
28
54
|
end
|
29
55
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HireFire
|
4
4
|
module Macro
|
@@ -13,6 +13,7 @@ module HireFire
|
|
13
13
|
# HireFire::Macro::Sidekiq.queue("audio", "video") # audio and video queues
|
14
14
|
# HireFire::Macro::Sidekiq.queue("email", skip_scheduled: true) # only email, will not count scheduled queue
|
15
15
|
# HireFire::Macro::Sidekiq.queue("audio", skip_retries: true) # only audio, will not count the retries queue
|
16
|
+
# HireFire::Macro::Sidekiq.queue("audio", skip_working: true) # only audio, will not count already queued
|
16
17
|
#
|
17
18
|
# @param [Array] queues provide one or more queue names, or none for "all".
|
18
19
|
# @return [Integer] the number of jobs in the queue(s).
|
@@ -60,7 +61,9 @@ module HireFire
|
|
60
61
|
in_retry = ::Sidekiq.redis { |c| c.zcount('retry', '-inf', Time.now.to_f) }
|
61
62
|
end
|
62
63
|
|
63
|
-
|
64
|
+
if !options[:skip_working]
|
65
|
+
in_progress = stats.workers_size
|
66
|
+
end
|
64
67
|
|
65
68
|
[in_queues, in_schedule, in_retry, in_progress].compact.inject(&:+)
|
66
69
|
end
|
@@ -96,15 +99,16 @@ module HireFire
|
|
96
99
|
|
97
100
|
now = Time.now.to_i
|
98
101
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
102
|
+
if !options[:skip_working]
|
103
|
+
# Objects yielded to Workers#each:
|
104
|
+
# https://github.com/mperham/sidekiq/blob/305ab8eedc362325da2e218b2a0e20e510668a42/lib/sidekiq/api.rb#L912
|
105
|
+
in_progress = ::Sidekiq::Workers.new.select do |key, tid, job|
|
106
|
+
queues.include?(job['queue']) && job['run_at'] <= now
|
107
|
+
end.size
|
108
|
+
end
|
104
109
|
|
105
110
|
[in_queues, in_schedule, in_retry, in_progress].compact.inject(&:+)
|
106
111
|
end
|
107
112
|
end
|
108
113
|
end
|
109
114
|
end
|
110
|
-
|
data/lib/hirefire/middleware.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HireFire
|
4
4
|
class Middleware
|
@@ -19,7 +19,7 @@ module HireFire
|
|
19
19
|
#
|
20
20
|
# When HireFire::Resource.log_queue_metrics is enabled, and the HTTP_X_REQUEST_START
|
21
21
|
# header has been injected at the Heroku Router layer, queue time information will be
|
22
|
-
# logged to
|
22
|
+
# logged to $stdout. This data can be used by the HireFire Logdrain with the
|
23
23
|
# Web.Logplex.QueueTime autoscaling strategy.
|
24
24
|
#
|
25
25
|
# Important: Don't set/update instance variables within this- or any underlying methods.
|
@@ -103,7 +103,7 @@ module HireFire
|
|
103
103
|
[status, headers, [body]]
|
104
104
|
end
|
105
105
|
|
106
|
-
# Writes the Heroku Router queue time to
|
106
|
+
# Writes the Heroku Router queue time to $stdout if a String was provided.
|
107
107
|
#
|
108
108
|
# @param [String] the timestamp from HTTP_X_REQUEST_START.
|
109
109
|
#
|
@@ -111,12 +111,12 @@ module HireFire
|
|
111
111
|
HireFire::Resource.log_queue_metrics && value && log_queue(value)
|
112
112
|
end
|
113
113
|
|
114
|
-
# Writes the Heroku Router queue time to
|
114
|
+
# Writes the Heroku Router queue time to $stdout.
|
115
115
|
#
|
116
116
|
# @param [String] the timestamp from HTTP_X_REQUEST_START.
|
117
117
|
#
|
118
118
|
def log_queue(value)
|
119
|
-
|
119
|
+
puts("[hirefire:router] queue=#{get_queue(value)}ms")
|
120
120
|
end
|
121
121
|
|
122
122
|
# Calculates the difference, in milliseconds, between the
|
data/lib/hirefire/railtie.rb
CHANGED
data/lib/hirefire/resource.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
|
-
#
|
1
|
+
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HireFire
|
4
4
|
module Resource
|
5
5
|
extend self
|
6
6
|
|
7
|
-
# This option, when enabled, will write queue metrics to
|
7
|
+
# This option, when enabled, will write queue metrics to $stdout,
|
8
8
|
# and is only required when using the Web.Logplex.QueueTime strategy.
|
9
9
|
#
|
10
10
|
# @param [Boolean] Whether or not the queue metrics should be logged.
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hirefire-resource
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.9.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael van Rooijen
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-10-17 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Load- and schedule-based scaling for web- and worker dynos
|
14
14
|
email: michael@hirefire.io
|
@@ -30,6 +30,7 @@ files:
|
|
30
30
|
- lib/hirefire/cli.rb
|
31
31
|
- lib/hirefire/macro/bunny.rb
|
32
32
|
- lib/hirefire/macro/delayed_job.rb
|
33
|
+
- lib/hirefire/macro/good_job.rb
|
33
34
|
- lib/hirefire/macro/qc.rb
|
34
35
|
- lib/hirefire/macro/qu.rb
|
35
36
|
- lib/hirefire/macro/que.rb
|
@@ -41,8 +42,12 @@ files:
|
|
41
42
|
homepage: https://www.hirefire.io
|
42
43
|
licenses:
|
43
44
|
- Apache License
|
44
|
-
metadata:
|
45
|
-
|
45
|
+
metadata:
|
46
|
+
homepage_uri: https://www.hirefire.io
|
47
|
+
changelog_uri: https://github.com/hirefire/hirefire-resource/blob/master/CHANGELOG.md
|
48
|
+
source_code_uri: https://github.com/hirefire/hirefire-resource/
|
49
|
+
bug_tracker_uri: https://github.com/hirefire/hirefire-resource/issues
|
50
|
+
post_install_message:
|
46
51
|
rdoc_options: []
|
47
52
|
require_paths:
|
48
53
|
- lib
|
@@ -57,8 +62,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
57
62
|
- !ruby/object:Gem::Version
|
58
63
|
version: '0'
|
59
64
|
requirements: []
|
60
|
-
rubygems_version: 3.
|
61
|
-
signing_key:
|
65
|
+
rubygems_version: 3.1.2
|
66
|
+
signing_key:
|
62
67
|
specification_version: 4
|
63
68
|
summary: Autoscaling for your Heroku dynos
|
64
69
|
test_files: []
|