sidekiq-heroku-scaler 0.2.0 → 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +2 -1
- data/Gemfile.lock +53 -36
- data/lib/sidekiq-heroku-scaler/heroku/client.rb +3 -5
- data/lib/sidekiq-heroku-scaler/sidekiq_config.rb +128 -0
- data/lib/sidekiq-heroku-scaler/strategy/latency.rb +6 -3
- data/lib/sidekiq-heroku-scaler/version.rb +1 -1
- data/lib/sidekiq-heroku-scaler/worker.rb +12 -1
- data/lib/sidekiq-heroku-scaler.rb +3 -0
- data/sidekiq-heroku-scaler.gemspec +2 -0
- metadata +26 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0e9d0725ad4a79e3107e26f97d01897d5a8358ebcfee6ad22788887abc623672
|
4
|
+
data.tar.gz: be22f33c68e8ca59796be0da4c7500afeac1d2189b14b698b9e23b8430ff3b1c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 47309241683be4547ca59d11acdf2c71d12c48ccb7f3d9778bae93176643295fb601e6c8c50d4f60edfeee8c33d09eac5947083d6d9948702d8a35b1b66ab50b
|
7
|
+
data.tar.gz: 8d08d0324d4b3315b28ba8629f993aeb009b4120ae5f06395976a8154571e03b2bfe05c17705f2cd8b6b9268fc9b4f24e06b7e34e3d7e3ed2722005545603b23
|
data/.rubocop.yml
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,67 +1,84 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
sidekiq-heroku-scaler (0.
|
4
|
+
sidekiq-heroku-scaler (0.3.4)
|
5
|
+
activesupport (> 5, < 8)
|
5
6
|
platform-api (> 3, < 4)
|
6
7
|
sidekiq (> 4, < 7)
|
7
8
|
|
8
9
|
GEM
|
9
10
|
remote: https://rubygems.org/
|
10
11
|
specs:
|
11
|
-
|
12
|
-
|
13
|
-
|
12
|
+
activesupport (7.0.4)
|
13
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
14
|
+
i18n (>= 1.6, < 2)
|
15
|
+
minitest (>= 5.1)
|
16
|
+
tzinfo (~> 2.0)
|
17
|
+
ast (2.4.2)
|
18
|
+
concurrent-ruby (1.1.10)
|
19
|
+
connection_pool (2.3.0)
|
20
|
+
diff-lcs (1.5.0)
|
14
21
|
erubis (2.7.0)
|
15
|
-
excon (0.92.
|
22
|
+
excon (0.92.4)
|
16
23
|
heroics (0.1.2)
|
17
24
|
erubis (~> 2.0)
|
18
25
|
excon
|
19
26
|
moneta
|
20
27
|
multi_json (>= 1.9.2)
|
21
28
|
webrick
|
22
|
-
|
29
|
+
i18n (1.12.0)
|
30
|
+
concurrent-ruby (~> 1.0)
|
31
|
+
json (2.6.2)
|
32
|
+
minitest (5.16.3)
|
23
33
|
moneta (1.0.0)
|
24
34
|
multi_json (1.15.0)
|
25
|
-
parallel (1.
|
26
|
-
parser (
|
27
|
-
ast (~> 2.4.
|
28
|
-
platform-api (3.
|
35
|
+
parallel (1.22.1)
|
36
|
+
parser (3.1.2.1)
|
37
|
+
ast (~> 2.4.1)
|
38
|
+
platform-api (3.5.0)
|
29
39
|
heroics (~> 0.1.1)
|
30
40
|
moneta (~> 1.0.0)
|
31
41
|
rate_throttle_client (~> 0.1.0)
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
rake (12.3.2)
|
42
|
+
rack (2.2.4)
|
43
|
+
rainbow (3.1.1)
|
44
|
+
rake (13.0.6)
|
36
45
|
rate_throttle_client (0.1.2)
|
37
|
-
redis (4.
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
rspec-
|
42
|
-
|
43
|
-
rspec-
|
44
|
-
rspec-
|
46
|
+
redis (4.8.0)
|
47
|
+
regexp_parser (2.5.0)
|
48
|
+
rexml (3.2.5)
|
49
|
+
rspec (3.11.0)
|
50
|
+
rspec-core (~> 3.11.0)
|
51
|
+
rspec-expectations (~> 3.11.0)
|
52
|
+
rspec-mocks (~> 3.11.0)
|
53
|
+
rspec-core (3.11.0)
|
54
|
+
rspec-support (~> 3.11.0)
|
55
|
+
rspec-expectations (3.11.1)
|
45
56
|
diff-lcs (>= 1.2.0, < 2.0)
|
46
|
-
rspec-support (~> 3.
|
47
|
-
rspec-mocks (3.
|
57
|
+
rspec-support (~> 3.11.0)
|
58
|
+
rspec-mocks (3.11.1)
|
48
59
|
diff-lcs (>= 1.2.0, < 2.0)
|
49
|
-
rspec-support (~> 3.
|
50
|
-
rspec-support (3.
|
51
|
-
rubocop (
|
52
|
-
|
60
|
+
rspec-support (~> 3.11.0)
|
61
|
+
rspec-support (3.11.1)
|
62
|
+
rubocop (1.36.0)
|
63
|
+
json (~> 2.3)
|
53
64
|
parallel (~> 1.10)
|
54
|
-
parser (>=
|
55
|
-
powerpack (~> 0.1)
|
65
|
+
parser (>= 3.1.2.1)
|
56
66
|
rainbow (>= 2.2.2, < 4.0)
|
67
|
+
regexp_parser (>= 1.8, < 3.0)
|
68
|
+
rexml (>= 3.2.5, < 4.0)
|
69
|
+
rubocop-ast (>= 1.20.1, < 2.0)
|
57
70
|
ruby-progressbar (~> 1.7)
|
58
|
-
unicode-display_width (
|
59
|
-
|
60
|
-
|
61
|
-
|
71
|
+
unicode-display_width (>= 1.4.0, < 3.0)
|
72
|
+
rubocop-ast (1.21.0)
|
73
|
+
parser (>= 3.1.1.0)
|
74
|
+
ruby-progressbar (1.11.0)
|
75
|
+
sidekiq (6.5.7)
|
76
|
+
connection_pool (>= 2.2.5)
|
62
77
|
rack (~> 2.0)
|
63
|
-
redis (>= 4.
|
64
|
-
|
78
|
+
redis (>= 4.5.0, < 5)
|
79
|
+
tzinfo (2.0.5)
|
80
|
+
concurrent-ruby (~> 1.0)
|
81
|
+
unicode-display_width (2.3.0)
|
65
82
|
webrick (1.7.0)
|
66
83
|
|
67
84
|
PLATFORMS
|
@@ -17,15 +17,13 @@ module SidekiqHerokuScaler
|
|
17
17
|
|
18
18
|
def formation_for(worker_name)
|
19
19
|
SidekiqHerokuScaler::Heroku::Formation.new(
|
20
|
-
formations.detect { |formation| formation['type'] == worker_name } || {}
|
20
|
+
formations.detect { |formation| formation['type'] == worker_name.to_s } || {}
|
21
21
|
)
|
22
22
|
end
|
23
23
|
|
24
24
|
def sidekiq_workers
|
25
|
-
@sidekiq_workers ||=
|
26
|
-
|
27
|
-
.map { |formation| formation['type'] }
|
28
|
-
end
|
25
|
+
@sidekiq_workers ||= formations.select { |formation| formation['command'].match(/sidekiq/) }
|
26
|
+
.map { |formation| formation['type'] }
|
29
27
|
end
|
30
28
|
|
31
29
|
def update_formation(formation_id, quantity)
|
@@ -0,0 +1,128 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'erb'
|
4
|
+
require 'fileutils'
|
5
|
+
require 'optparse'
|
6
|
+
require 'sidekiq'
|
7
|
+
|
8
|
+
module SidekiqHerokuScaler
|
9
|
+
class SidekiqConfig
|
10
|
+
def initialize(commands = [])
|
11
|
+
@commands = commands
|
12
|
+
end
|
13
|
+
|
14
|
+
def config
|
15
|
+
@config ||= ActiveSupport::HashWithIndifferentAccess.new(setup_options(commands))
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
|
20
|
+
attr_reader :commands
|
21
|
+
|
22
|
+
def load_yaml(src)
|
23
|
+
if Psych::VERSION > '4.0'
|
24
|
+
YAML.safe_load(src, permitted_classes: [Symbol], aliases: true)
|
25
|
+
else
|
26
|
+
YAML.safe_load(src)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def option_parser(opts)
|
31
|
+
OptionParser.new do |o|
|
32
|
+
o.on '-c', '--concurrency INT', 'processor threads to use' do |arg|
|
33
|
+
opts[:concurrency] = Integer(arg)
|
34
|
+
end
|
35
|
+
|
36
|
+
o.on '-g', '--tag TAG', 'Process tag for procline' do |arg|
|
37
|
+
opts[:tag] = arg
|
38
|
+
end
|
39
|
+
|
40
|
+
o.on '-q', '--queue QUEUE[,WEIGHT]', 'Queues to process with optional weights' do |arg|
|
41
|
+
queue, weight = arg.split(',')
|
42
|
+
parse_queue opts, queue, weight
|
43
|
+
end
|
44
|
+
|
45
|
+
o.on '-r', '--require [PATH|DIR]', 'Location of Rails application with jobs or file to require' do |arg|
|
46
|
+
opts[:require] = arg
|
47
|
+
end
|
48
|
+
|
49
|
+
o.on '-C', '--config PATH', 'path to YAML config file' do |arg|
|
50
|
+
opts[:config_file] = arg
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def parse_config(path)
|
56
|
+
erb = ERB.new(File.read(path))
|
57
|
+
erb.filename = File.expand_path(path)
|
58
|
+
opts = load_yaml(erb.result) || {}
|
59
|
+
|
60
|
+
if opts.respond_to?(:deep_symbolize_keys!)
|
61
|
+
opts.deep_symbolize_keys!
|
62
|
+
else
|
63
|
+
symbolize_keys_deep!(opts)
|
64
|
+
end
|
65
|
+
|
66
|
+
opts.delete(:strict)
|
67
|
+
|
68
|
+
parse_queues(opts, opts.delete(:queues) || [])
|
69
|
+
|
70
|
+
opts
|
71
|
+
end
|
72
|
+
|
73
|
+
def parse_queues(opts, queues_and_weights)
|
74
|
+
queues_and_weights.each { |queue_and_weight| parse_queue(opts, *queue_and_weight) }
|
75
|
+
end
|
76
|
+
|
77
|
+
def parse_queue(opts, queue, weight = nil)
|
78
|
+
opts[:queues] ||= []
|
79
|
+
opts[:strict] = true if opts[:strict].nil?
|
80
|
+
raise ArgumentError, "queues: #{queue} cannot be defined twice" if opts[:queues].include?(queue)
|
81
|
+
|
82
|
+
opts[:queues] << queue.to_s
|
83
|
+
opts[:strict] = false if weight.to_i.positive?
|
84
|
+
end
|
85
|
+
|
86
|
+
def parse_options(argv)
|
87
|
+
opts = {}
|
88
|
+
parser = option_parser(opts)
|
89
|
+
parser.parse!(argv)
|
90
|
+
opts
|
91
|
+
end
|
92
|
+
|
93
|
+
def setup_options(args)
|
94
|
+
# parse CLI options
|
95
|
+
opts = parse_options(args)
|
96
|
+
|
97
|
+
unless opts[:config_file]
|
98
|
+
config_dir = if File.directory?(opts[:require].to_s)
|
99
|
+
File.join(opts[:require], 'config')
|
100
|
+
else
|
101
|
+
File.join(Sidekiq[:require], 'config')
|
102
|
+
end
|
103
|
+
|
104
|
+
%w[sidekiq.yml sidekiq.yml.erb].each do |config_file|
|
105
|
+
path = File.join(config_dir, config_file)
|
106
|
+
opts[:config_file] ||= path if File.exist?(path)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
# parse config file options
|
111
|
+
opts = parse_config(opts[:config_file]).merge(opts) if opts[:config_file] && File.exist?(opts[:config_file])
|
112
|
+
|
113
|
+
# set defaults
|
114
|
+
opts[:queues] = ['default'] if opts[:queues].nil?
|
115
|
+
opts[:concurrency] = Integer(ENV['RAILS_MAX_THREADS']) if opts[:concurrency].nil? && ENV['RAILS_MAX_THREADS']
|
116
|
+
|
117
|
+
opts
|
118
|
+
end
|
119
|
+
|
120
|
+
def symbolize_keys_deep!(hash)
|
121
|
+
hash.each_key do |k|
|
122
|
+
symkey = k.respond_to?(:to_sym) ? k.to_sym : k
|
123
|
+
hash[symkey] = hash.delete k
|
124
|
+
symbolize_keys_deep! hash[symkey] if hash[symkey].is_a? Hash
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
end
|
@@ -18,16 +18,19 @@ module SidekiqHerokuScaler
|
|
18
18
|
|
19
19
|
def increase?(sidekiq_worker)
|
20
20
|
sidekiq_worker.quantity < max_dynos_count &&
|
21
|
-
sidekiq_worker.latency > max_latency
|
21
|
+
(sidekiq_worker.latency > max_latency ||
|
22
|
+
(sidekiq_worker.quantity.zero? && sidekiq_worker.latency.positive?)) &&
|
22
23
|
sidekiq_worker.queues_size > sidekiq_worker.quantity * sidekiq_worker.concurrency
|
23
24
|
end
|
24
25
|
|
25
26
|
def decrease?(sidekiq_worker)
|
26
|
-
sidekiq_worker.
|
27
|
+
sidekiq_worker.quantity > min_dynos_count &&
|
28
|
+
sidekiq_worker.latency < min_latency &&
|
29
|
+
(sidekiq_worker.quantity > 1 || !sidekiq_worker.jobs_running?)
|
27
30
|
end
|
28
31
|
|
29
32
|
def safe_quantity(quantity)
|
30
|
-
return
|
33
|
+
return min_dynos_count if quantity < min_dynos_count
|
31
34
|
|
32
35
|
quantity > max_dynos_count ? max_dynos_count : quantity
|
33
36
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'sidekiq'
|
4
|
+
require 'sidekiq-heroku-scaler/sidekiq_config'
|
4
5
|
|
5
6
|
module SidekiqHerokuScaler
|
6
7
|
class Worker
|
@@ -21,6 +22,10 @@ module SidekiqHerokuScaler
|
|
21
22
|
formation.quantity
|
22
23
|
end
|
23
24
|
|
25
|
+
def jobs_running?
|
26
|
+
Sidekiq::Workers.new.any? { |_process_id, _thread_id, work| queues.include?(work['queue']) }
|
27
|
+
end
|
28
|
+
|
24
29
|
def latency
|
25
30
|
queues.sum { |queue| Sidekiq::Queue.new(queue).latency }
|
26
31
|
end
|
@@ -33,6 +38,12 @@ module SidekiqHerokuScaler
|
|
33
38
|
|
34
39
|
attr_reader :formation, :worker_name
|
35
40
|
|
41
|
+
def build_process
|
42
|
+
command = formation.command.gsub(/.*sidekiq(\s|\z)/, '').split
|
43
|
+
sideki_config = SidekiqHerokuScaler::SidekiqConfig.new(command)
|
44
|
+
Sidekiq::Process.new(sideki_config.config)
|
45
|
+
end
|
46
|
+
|
36
47
|
def queues
|
37
48
|
process['queues'] || []
|
38
49
|
end
|
@@ -42,7 +53,7 @@ module SidekiqHerokuScaler
|
|
42
53
|
end
|
43
54
|
|
44
55
|
def process
|
45
|
-
process_set.detect { |p| p.identity.match(/\A#{worker_name}\./) } ||
|
56
|
+
@process ||= process_set.detect { |p| p.identity.match(/\A#{worker_name}\./) } || build_process
|
46
57
|
end
|
47
58
|
end
|
48
59
|
end
|
@@ -27,6 +27,8 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.add_development_dependency 'rspec'
|
28
28
|
spec.add_development_dependency 'rubocop'
|
29
29
|
|
30
|
+
spec.add_dependency 'activesupport', '> 5', '< 8'
|
30
31
|
spec.add_dependency 'platform-api', '> 3', '< 4'
|
31
32
|
spec.add_dependency 'sidekiq', '> 4', '< 7'
|
33
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
32
34
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-heroku-scaler
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Vital Ryabchinskiy
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-09-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -66,6 +66,26 @@ dependencies:
|
|
66
66
|
- - ">="
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activesupport
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">"
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5'
|
76
|
+
- - "<"
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
version: '8'
|
79
|
+
type: :runtime
|
80
|
+
prerelease: false
|
81
|
+
version_requirements: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ">"
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '5'
|
86
|
+
- - "<"
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '8'
|
69
89
|
- !ruby/object:Gem::Dependency
|
70
90
|
name: platform-api
|
71
91
|
requirement: !ruby/object:Gem::Requirement
|
@@ -128,6 +148,7 @@ files:
|
|
128
148
|
- lib/sidekiq-heroku-scaler/heroku/client.rb
|
129
149
|
- lib/sidekiq-heroku-scaler/heroku/formation.rb
|
130
150
|
- lib/sidekiq-heroku-scaler/manager.rb
|
151
|
+
- lib/sidekiq-heroku-scaler/sidekiq_config.rb
|
131
152
|
- lib/sidekiq-heroku-scaler/strategy/latency.rb
|
132
153
|
- lib/sidekiq-heroku-scaler/version.rb
|
133
154
|
- lib/sidekiq-heroku-scaler/worker.rb
|
@@ -135,7 +156,8 @@ files:
|
|
135
156
|
homepage: https://github.com/vitalinfo/sidekiq-heroku-scaler
|
136
157
|
licenses:
|
137
158
|
- MIT
|
138
|
-
metadata:
|
159
|
+
metadata:
|
160
|
+
rubygems_mfa_required: 'true'
|
139
161
|
post_install_message:
|
140
162
|
rdoc_options: []
|
141
163
|
require_paths:
|
@@ -151,7 +173,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
151
173
|
- !ruby/object:Gem::Version
|
152
174
|
version: '0'
|
153
175
|
requirements: []
|
154
|
-
rubygems_version: 3.
|
176
|
+
rubygems_version: 3.3.7
|
155
177
|
signing_key:
|
156
178
|
specification_version: 4
|
157
179
|
summary: Sidekiq Heroku Scaler
|