delayed_job_worker_pool 0.2.2 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/.circleci/config.yml +80 -0
- data/.github/CODEOWNERS +1 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +8 -0
- data/Appraisals +13 -0
- data/CHANGELOG.md +13 -1
- data/Gemfile +2 -0
- data/README.md +28 -6
- data/Rakefile +2 -0
- data/bin/delayed_job_worker_pool +1 -0
- data/delayed_job_worker_pool.gemspec +19 -7
- data/gemfiles/rails_6.0.gemfile +7 -0
- data/gemfiles/rails_6.1.gemfile +7 -0
- data/gemfiles/rails_7.0.gemfile +7 -0
- data/lib/delayed_job_worker_pool/application.rb +3 -1
- data/lib/delayed_job_worker_pool/dsl.rb +24 -6
- data/lib/delayed_job_worker_pool/registry.rb +63 -0
- data/lib/delayed_job_worker_pool/version.rb +3 -1
- data/lib/delayed_job_worker_pool/worker.rb +2 -0
- data/lib/delayed_job_worker_pool/worker_group_options.rb +17 -0
- data/lib/delayed_job_worker_pool/worker_info.rb +4 -2
- data/lib/delayed_job_worker_pool/worker_pool.rb +60 -37
- data/lib/delayed_job_worker_pool.rb +4 -0
- metadata +91 -20
- data/.travis.yml +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ec3712fc2bae4eaac3d8d07ab408616c2efef0763d0d173ff65281d267a42f30
|
4
|
+
data.tar.gz: cf6f8866ad4de83cd52e110d5f8d69578af11fa3ba6397448b32400ba711a048
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e095fcb89fa76967608d5d4e513cce17034d209d24f03b7531a39d25843a1e8bc80923a2b0a5de66d4e804c0912af853622edde7e441da19b9a52c84b991266
|
7
|
+
data.tar.gz: c1fe489f6a6172a3d61f7ad27fd384576cda69cbc0160a49bcf393df813d245e18614ba3321d14d18d5284318f18751db220be193deb2712713d2517722756de
|
@@ -0,0 +1,80 @@
|
|
1
|
+
version: 2.1
|
2
|
+
jobs:
|
3
|
+
lint:
|
4
|
+
docker:
|
5
|
+
- image: salsify/ruby_ci:2.7.6
|
6
|
+
working_directory: ~/delayed_job_worker_pool
|
7
|
+
steps:
|
8
|
+
- checkout
|
9
|
+
- restore_cache:
|
10
|
+
keys:
|
11
|
+
- v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
|
12
|
+
- v1-gems-ruby-2.7.6-
|
13
|
+
- run:
|
14
|
+
name: Install Gems
|
15
|
+
command: |
|
16
|
+
if ! bundle check --path=vendor/bundle; then
|
17
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
18
|
+
bundle clean
|
19
|
+
fi
|
20
|
+
- save_cache:
|
21
|
+
key: v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
|
22
|
+
paths:
|
23
|
+
- "vendor/bundle"
|
24
|
+
- "gemfiles/vendor/bundle"
|
25
|
+
- run:
|
26
|
+
name: Run Rubocop
|
27
|
+
command: bundle exec rubocop --config .rubocop.yml
|
28
|
+
test:
|
29
|
+
parameters:
|
30
|
+
ruby_version:
|
31
|
+
type: string
|
32
|
+
gemfile:
|
33
|
+
type: string
|
34
|
+
docker:
|
35
|
+
- image: salsify/ruby_ci:<< parameters.ruby_version >>
|
36
|
+
environment:
|
37
|
+
CIRCLE_TEST_REPORTS: "test-results"
|
38
|
+
BUNDLE_GEMFILE: "~/delayed_job_worker_pool/<< parameters.gemfile >>"
|
39
|
+
working_directory: ~/delayed_job_worker_pool
|
40
|
+
steps:
|
41
|
+
- checkout
|
42
|
+
- restore_cache:
|
43
|
+
keys:
|
44
|
+
- v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
45
|
+
- v1-gems-ruby-<< parameters.ruby_version >>-
|
46
|
+
- run:
|
47
|
+
name: Install Gems
|
48
|
+
command: |
|
49
|
+
if ! bundle check --path=vendor/bundle; then
|
50
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
51
|
+
bundle clean
|
52
|
+
fi
|
53
|
+
- save_cache:
|
54
|
+
key: v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
55
|
+
paths:
|
56
|
+
- "vendor/bundle"
|
57
|
+
- "gemfiles/vendor/bundle"
|
58
|
+
- run:
|
59
|
+
name: Run Tests
|
60
|
+
command: |
|
61
|
+
bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
|
62
|
+
- store_test_results:
|
63
|
+
path: "test-results"
|
64
|
+
- store_artifacts:
|
65
|
+
path: "tmp/log/"
|
66
|
+
workflows:
|
67
|
+
build:
|
68
|
+
jobs:
|
69
|
+
- lint
|
70
|
+
- test:
|
71
|
+
matrix:
|
72
|
+
parameters:
|
73
|
+
gemfile:
|
74
|
+
- "gemfiles/rails_6.0.gemfile"
|
75
|
+
- "gemfiles/rails_6.1.gemfile"
|
76
|
+
- "gemfiles/rails_7.0.gemfile"
|
77
|
+
ruby_version:
|
78
|
+
- "2.7.6"
|
79
|
+
- "3.0.4"
|
80
|
+
- "3.1.2"
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @jturkel @will89
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/Appraisals
ADDED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,18 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
### 0.
|
3
|
+
### 1.0.0
|
4
|
+
* Require Ruby 2.7 or higher.
|
5
|
+
|
6
|
+
### 0.3.0
|
7
|
+
* Require Ruby 2.5 or higher.
|
8
|
+
* Support for running multiple worker pools on a single node.
|
9
|
+
See [#7](https://github.com/salsify/delayed_job_worker_pool/pull/7) for details.
|
10
|
+
Thanks to Severin Räz!
|
11
|
+
|
12
|
+
### 0.2.3
|
13
|
+
* Explicitly require 'fcntl' to fix uninitialized constant IO::Fcntl. Thanks to Stefan Wrobel!
|
14
|
+
|
15
|
+
### 0.2.2
|
4
16
|
* Add support for Delayed Job 4.1
|
5
17
|
|
6
18
|
### 0.2.1
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -41,9 +41,11 @@ delayed_job_worker_pool <config file>
|
|
41
41
|
The config file is a Ruby DSL inspired by the [Puma](https://github.com/puma/puma) configuration DSL. Here's an example:
|
42
42
|
|
43
43
|
```ruby
|
44
|
-
|
45
|
-
|
46
|
-
|
44
|
+
worker_group do |g|
|
45
|
+
g.workers = Integer(ENV['NUM_WORKERS'] || 1)
|
46
|
+
g.queues = (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
|
47
|
+
g.sleep_delay = ENV['WORKER_SLEEP_DELAY']
|
48
|
+
end
|
47
49
|
|
48
50
|
preload_app
|
49
51
|
|
@@ -77,10 +79,26 @@ after_worker_shutdown do |worker_info|
|
|
77
79
|
end
|
78
80
|
```
|
79
81
|
|
82
|
+
You can configure multiple worker groups, i.e.:
|
83
|
+
|
84
|
+
```
|
85
|
+
worker_group(:default) do |g|
|
86
|
+
g.workers = 1
|
87
|
+
g.queues = ['default']
|
88
|
+
end
|
89
|
+
|
90
|
+
worker_group(:mails) do |g|
|
91
|
+
g.workers = 1
|
92
|
+
g.queues = ['mail']
|
93
|
+
end
|
94
|
+
|
95
|
+
```
|
96
|
+
|
80
97
|
Here's more information on each setting:
|
81
98
|
|
82
|
-
* `
|
83
|
-
*
|
99
|
+
* `worker_group` - You need at least one worker group. Group settings can be set as illustrated above. Worker group settings:
|
100
|
+
* `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
|
101
|
+
* Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
|
84
102
|
* `preload_app` - This forces the master process to load Rails before forking worker processes causing the memory consumed by the code to be shared between workers. **If you use this setting make sure you re-establish any necessary connections in the on_worker_boot callback.**
|
85
103
|
* `after_preload_app` - A callback that runs in the master process after preloading the app but before forking any workers.
|
86
104
|
* `on_worker_boot` - A callback that runs in the worker process after it has been forked.
|
@@ -89,11 +107,15 @@ Here's more information on each setting:
|
|
89
107
|
|
90
108
|
All settings are optional and nil values are ignored.
|
91
109
|
|
110
|
+
## Upgrading from v0.2.x
|
111
|
+
|
112
|
+
* Convert your worker settings to a single worker group (see _Usage_)
|
113
|
+
* Please note the delayed job worker names changed to include ` group: <group_name>`, e.g. if you are monitoring them by their name
|
114
|
+
|
92
115
|
## Contributing
|
93
116
|
|
94
117
|
Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_worker_pool.
|
95
118
|
|
96
|
-
|
97
119
|
## License
|
98
120
|
|
99
121
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
CHANGED
data/bin/delayed_job_worker_pool
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'delayed_job_worker_pool/version'
|
5
6
|
|
@@ -13,18 +14,29 @@ Gem::Specification.new do |spec|
|
|
13
14
|
spec.homepage = 'https://github.com/salsify/delayed_job_worker_pool'
|
14
15
|
spec.license = 'MIT'
|
15
16
|
|
17
|
+
if spec.respond_to?(:metadata)
|
18
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
19
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
20
|
+
else
|
21
|
+
raise 'RubyGems 2.0 or newer is required to set allowed_push_host.'
|
22
|
+
end
|
23
|
+
|
16
24
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
25
|
spec.executables = ['delayed_job_worker_pool']
|
18
26
|
spec.require_paths = ['lib']
|
19
27
|
|
20
|
-
spec.required_ruby_version = '>= 2.
|
28
|
+
spec.required_ruby_version = '>= 2.7'
|
21
29
|
|
22
30
|
spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.2']
|
23
31
|
|
32
|
+
spec.add_development_dependency 'appraisal'
|
33
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
24
34
|
spec.add_development_dependency 'delayed_job_active_record'
|
25
|
-
spec.add_development_dependency '
|
26
|
-
spec.add_development_dependency 'rake', '~>
|
27
|
-
spec.add_development_dependency 'rspec', '>= 3.
|
35
|
+
spec.add_development_dependency 'rails', '>= 6.0', '< 8'
|
36
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
37
|
+
spec.add_development_dependency 'rspec', '>= 3.8'
|
38
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
39
|
+
spec.add_development_dependency 'salsify_rubocop', '~> 1.27.1'
|
40
|
+
spec.add_development_dependency 'sprockets', '< 4'
|
28
41
|
spec.add_development_dependency 'sqlite3', '>= 1.3'
|
29
|
-
spec.add_development_dependency 'rails', '>= 4.2'
|
30
42
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
module Application
|
3
5
|
extend self
|
@@ -6,7 +8,7 @@ module DelayedJobWorkerPool
|
|
6
8
|
require(base_application_filename)
|
7
9
|
rescue LoadError
|
8
10
|
raise "Could not find Rails initialization file #{full_application_filename}. " \
|
9
|
-
|
11
|
+
'Make sure delayed_job_worker_pool is run from the Rails root directory.'
|
10
12
|
end
|
11
13
|
|
12
14
|
private
|
@@ -1,29 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
class DSL
|
3
|
-
|
5
|
+
class NoWorkerGroupsDefined < StandardError; end
|
6
|
+
class NonUniqueGroupName < StandardError; end
|
7
|
+
|
4
8
|
CALLBACK_SETTINGS = [:after_preload_app, :on_worker_boot, :after_worker_boot, :after_worker_shutdown].freeze
|
9
|
+
DEFAULT_WORKER_GROUP_NAME = :default
|
5
10
|
|
6
11
|
def self.load(path)
|
7
12
|
options = {}
|
8
13
|
|
9
14
|
dsl = new(options)
|
10
15
|
dsl.instance_eval(File.read(path), path, 1)
|
16
|
+
dsl.assert_groups_defined!
|
11
17
|
|
12
18
|
options
|
13
19
|
end
|
14
20
|
|
15
21
|
def initialize(options)
|
16
22
|
@options = options
|
23
|
+
@options[:worker_groups] ||= {}
|
24
|
+
end
|
25
|
+
|
26
|
+
def preload_app(preload = true)
|
27
|
+
@options[:preload_app] = preload
|
17
28
|
end
|
18
29
|
|
19
|
-
|
20
|
-
|
21
|
-
|
30
|
+
def worker_group(name = DEFAULT_WORKER_GROUP_NAME)
|
31
|
+
name_sym = name.to_sym
|
32
|
+
if @options[:worker_groups].key?(name_sym)
|
33
|
+
raise NonUniqueGroupName.new("Worker group name #{name_sym} is already in use")
|
22
34
|
end
|
35
|
+
|
36
|
+
group_options = WorkerGroupOptions.new
|
37
|
+
yield(group_options)
|
38
|
+
@options[:worker_groups][name_sym] = group_options
|
23
39
|
end
|
24
40
|
|
25
|
-
def
|
26
|
-
@options[:
|
41
|
+
def assert_groups_defined!
|
42
|
+
return unless @options[:worker_groups].empty?
|
43
|
+
|
44
|
+
raise NoWorkerGroupsDefined.new('No worker groups defined. Define groups using `worker_group`.')
|
27
45
|
end
|
28
46
|
|
29
47
|
CALLBACK_SETTINGS.each do |option_name|
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DelayedJobWorkerPool
|
4
|
+
# Keeps track of worker groups and their workers.
|
5
|
+
class Registry
|
6
|
+
class GroupAlreadyExists < StandardError; end
|
7
|
+
class GroupDoesNotExist < StandardError; end
|
8
|
+
class GroupNotFound < StandardError; end
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@groups = {}
|
12
|
+
end
|
13
|
+
|
14
|
+
def include_worker?(pid)
|
15
|
+
worker_pids.include?(pid)
|
16
|
+
end
|
17
|
+
|
18
|
+
def workers?
|
19
|
+
!worker_pids.empty?
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_group(name, options)
|
23
|
+
raise GroupAlreadyExists.new("Group #{group} already exists") if @groups.key?(name)
|
24
|
+
|
25
|
+
@groups[name] = {
|
26
|
+
options: options,
|
27
|
+
pids: []
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def add_worker(group_name, pid)
|
32
|
+
group_by_name(group_name)[:pids] << pid
|
33
|
+
end
|
34
|
+
|
35
|
+
def remove_worker(pid)
|
36
|
+
@groups[group(pid)][:pids].delete(pid)
|
37
|
+
end
|
38
|
+
|
39
|
+
def options(group_name)
|
40
|
+
group_by_name(group_name)[:options]
|
41
|
+
end
|
42
|
+
|
43
|
+
def worker_pids
|
44
|
+
@groups.values.flat_map { |v| v[:pids] }
|
45
|
+
end
|
46
|
+
|
47
|
+
def group(pid)
|
48
|
+
@groups.each do |name, group|
|
49
|
+
return name if group[:pids].include?(pid)
|
50
|
+
end
|
51
|
+
raise GroupNotFound.new("No group found for PID #{pid}")
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def group_by_name(name)
|
57
|
+
match = @groups[name]
|
58
|
+
return match unless match.nil?
|
59
|
+
|
60
|
+
raise GroupDoesNotExist.new("No group with name #{name.inspect} found")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DelayedJobWorkerPool
|
4
|
+
class WorkerGroupOptions
|
5
|
+
DJ_SETTINGS = [:queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
|
6
|
+
GROUP_SETTINGS = [:workers].freeze
|
7
|
+
|
8
|
+
attr_accessor *DJ_SETTINGS, *GROUP_SETTINGS
|
9
|
+
|
10
|
+
# @return an options hash for `Delayed::Worker`
|
11
|
+
def dj_worker_options
|
12
|
+
DJ_SETTINGS.each_with_object({}) do |setting, memo|
|
13
|
+
memo[setting] = send(setting)
|
14
|
+
end.compact
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -1,11 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
class WorkerInfo
|
3
|
-
attr_reader :process_id, :name
|
5
|
+
attr_reader :process_id, :name, :worker_group
|
4
6
|
|
5
7
|
def initialize(attributes)
|
6
8
|
@process_id = attributes.fetch(:process_id)
|
7
9
|
@name = attributes.fetch(:name)
|
10
|
+
@worker_group = attributes.fetch(:worker_group)
|
8
11
|
end
|
9
|
-
|
10
12
|
end
|
11
13
|
end
|
@@ -1,10 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fcntl'
|
4
|
+
require 'socket'
|
5
|
+
|
1
6
|
module DelayedJobWorkerPool
|
2
7
|
class WorkerPool
|
8
|
+
|
3
9
|
SIGNALS = ['TERM', 'INT'].map(&:freeze).freeze
|
10
|
+
DEFAULT_WORKER_COUNT = 1
|
4
11
|
|
5
12
|
def initialize(options = {})
|
6
13
|
@options = options
|
7
|
-
@
|
14
|
+
@registry = Registry.new
|
8
15
|
@pending_signals = []
|
9
16
|
@pending_signal_read_pipe, @pending_signal_write_pipe = create_pipe(inheritable: false)
|
10
17
|
@master_alive_read_pipe, @master_alive_write_pipe = create_pipe(inheritable: true)
|
@@ -23,7 +30,7 @@ module DelayedJobWorkerPool
|
|
23
30
|
|
24
31
|
log_uninheritable_threads
|
25
32
|
|
26
|
-
|
33
|
+
fork_workers
|
27
34
|
|
28
35
|
monitor_workers
|
29
36
|
|
@@ -35,7 +42,7 @@ module DelayedJobWorkerPool
|
|
35
42
|
|
36
43
|
private
|
37
44
|
|
38
|
-
attr_reader :options, :
|
45
|
+
attr_reader :options, :registry, :master_alive_read_pipe, :master_alive_write_pipe,
|
39
46
|
:pending_signals, :pending_signal_read_pipe, :pending_signal_write_pipe
|
40
47
|
attr_accessor :shutting_down
|
41
48
|
|
@@ -57,8 +64,10 @@ module DelayedJobWorkerPool
|
|
57
64
|
def log_uninheritable_threads
|
58
65
|
Thread.list.each do |t|
|
59
66
|
next if t == Thread.current
|
67
|
+
|
60
68
|
if t.respond_to?(:backtrace)
|
61
|
-
log("WARNING: Thread will not be inherited by workers: #{t.inspect} -
|
69
|
+
log("WARNING: Thread will not be inherited by workers: #{t.inspect} - " \
|
70
|
+
"#{t.backtrace ? t.backtrace.first : ''}")
|
62
71
|
else
|
63
72
|
log("WARNING: Thread will not be inherited by workers: #{t.inspect}")
|
64
73
|
end
|
@@ -72,15 +81,16 @@ module DelayedJobWorkerPool
|
|
72
81
|
def shutdown(signal)
|
73
82
|
log("Shutting down master #{Process.pid} with signal #{signal}")
|
74
83
|
self.shutting_down = true
|
75
|
-
worker_pids.each do |child_pid|
|
76
|
-
|
84
|
+
registry.worker_pids.each do |child_pid|
|
85
|
+
group = registry.group(child_pid)
|
86
|
+
log("Telling worker #{child_pid} from group #{group} to shutdown with signal #{signal}")
|
77
87
|
Process.kill(signal, child_pid)
|
78
88
|
end
|
79
89
|
end
|
80
90
|
|
81
91
|
def monitor_workers
|
82
|
-
while
|
83
|
-
if
|
92
|
+
while workers?
|
93
|
+
if pending_signal?
|
84
94
|
shutdown(pending_signals.pop)
|
85
95
|
elsif (wait_result = Process.wait2(-1, Process::WNOHANG))
|
86
96
|
handle_dead_worker(wait_result.first, wait_result.last)
|
@@ -91,19 +101,22 @@ module DelayedJobWorkerPool
|
|
91
101
|
end
|
92
102
|
|
93
103
|
def handle_dead_worker(worker_pid, status)
|
94
|
-
return unless
|
104
|
+
return unless registry.include_worker?(worker_pid)
|
95
105
|
|
96
106
|
log("Worker #{worker_pid} exited with status #{status.to_i}")
|
97
|
-
|
98
|
-
|
99
|
-
|
107
|
+
|
108
|
+
group = registry.group(worker_pid)
|
109
|
+
invoke_callback(:after_worker_shutdown, worker_info(worker_pid, group))
|
110
|
+
|
111
|
+
registry.remove_worker(worker_pid)
|
112
|
+
fork_worker(group) unless shutting_down
|
100
113
|
end
|
101
114
|
|
102
|
-
def
|
103
|
-
|
115
|
+
def workers?
|
116
|
+
registry.workers?
|
104
117
|
end
|
105
118
|
|
106
|
-
def
|
119
|
+
def pending_signal?
|
107
120
|
!pending_signals.empty?
|
108
121
|
end
|
109
122
|
|
@@ -111,14 +124,26 @@ module DelayedJobWorkerPool
|
|
111
124
|
options[callback_name].call(*args) if options[callback_name]
|
112
125
|
end
|
113
126
|
|
114
|
-
def
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
127
|
+
def fork_workers
|
128
|
+
options.fetch(:worker_groups).each do |name, group|
|
129
|
+
workers = group.workers || DEFAULT_WORKER_COUNT
|
130
|
+
|
131
|
+
registry.add_group(name, group.dj_worker_options)
|
132
|
+
|
133
|
+
workers.times { fork_worker(name) }
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def fork_worker(group)
|
138
|
+
worker_pid = Kernel.fork { run_worker(group) }
|
139
|
+
log("Started worker in group #{group}: #{worker_pid}")
|
140
|
+
|
141
|
+
registry.add_worker(group, worker_pid)
|
142
|
+
|
143
|
+
invoke_callback(:after_worker_boot, worker_info(worker_pid, group))
|
119
144
|
end
|
120
145
|
|
121
|
-
def run_worker
|
146
|
+
def run_worker(group)
|
122
147
|
master_alive_write_pipe.close
|
123
148
|
|
124
149
|
uninstall_signal_handlers
|
@@ -131,32 +156,32 @@ module DelayedJobWorkerPool
|
|
131
156
|
|
132
157
|
load_app unless preload_app?
|
133
158
|
|
134
|
-
invoke_callback(:on_worker_boot, worker_info(Process.pid))
|
159
|
+
invoke_callback(:on_worker_boot, worker_info(Process.pid, group))
|
135
160
|
|
136
|
-
DelayedJobWorkerPool::Worker.run(worker_options(Process.pid))
|
137
|
-
rescue => e
|
161
|
+
DelayedJobWorkerPool::Worker.run(worker_options(Process.pid, group))
|
162
|
+
rescue StandardError => e
|
138
163
|
log("Worker failed with error: #{e.message}\n#{e.backtrace.join("\n")}")
|
139
164
|
exit(1)
|
140
165
|
end
|
141
166
|
|
142
|
-
def worker_info(worker_pid)
|
143
|
-
DelayedJobWorkerPool::WorkerInfo.new(
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
167
|
+
def worker_info(worker_pid, group)
|
168
|
+
DelayedJobWorkerPool::WorkerInfo.new(
|
169
|
+
name: worker_name(worker_pid, group),
|
170
|
+
process_id: worker_pid,
|
171
|
+
worker_group: group
|
172
|
+
)
|
148
173
|
end
|
149
174
|
|
150
|
-
def
|
151
|
-
|
175
|
+
def worker_name(worker_pid, group)
|
176
|
+
"host:#{Socket.gethostname} pid:#{worker_pid} group:#{group}"
|
152
177
|
end
|
153
178
|
|
154
179
|
def preload_app?
|
155
180
|
options.fetch(:preload_app, false)
|
156
181
|
end
|
157
182
|
|
158
|
-
def worker_options(worker_pid)
|
159
|
-
options
|
183
|
+
def worker_options(worker_pid, group)
|
184
|
+
registry.options(group).merge(name: worker_name(worker_pid, group))
|
160
185
|
end
|
161
186
|
|
162
187
|
def create_pipe(inheritable: true)
|
@@ -173,9 +198,7 @@ module DelayedJobWorkerPool
|
|
173
198
|
end
|
174
199
|
|
175
200
|
def wait_for_signal(timeout)
|
176
|
-
if IO.select([pending_signal_read_pipe], [], [], timeout)
|
177
|
-
drain_pipe(pending_signal_read_pipe)
|
178
|
-
end
|
201
|
+
drain_pipe(pending_signal_read_pipe) if IO.select([pending_signal_read_pipe], [], [], timeout)
|
179
202
|
end
|
180
203
|
|
181
204
|
def drain_pipe(pipe)
|
@@ -1,6 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'delayed_job_worker_pool/application'
|
2
4
|
require 'delayed_job_worker_pool/dsl'
|
5
|
+
require 'delayed_job_worker_pool/registry'
|
3
6
|
require 'delayed_job_worker_pool/worker'
|
4
7
|
require 'delayed_job_worker_pool/worker_info'
|
5
8
|
require 'delayed_job_worker_pool/worker_pool'
|
9
|
+
require 'delayed_job_worker_pool/worker_group_options'
|
6
10
|
require 'delayed_job_worker_pool/version'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: delayed_job_worker_pool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joel Turkel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: delayed_job
|
@@ -31,7 +31,7 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '4.2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: appraisal
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
@@ -50,70 +50,132 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '2.0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '2.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: delayed_job_active_record
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: rails
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '6.0'
|
82
|
+
- - "<"
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '8'
|
85
|
+
type: :development
|
86
|
+
prerelease: false
|
87
|
+
version_requirements: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '6.0'
|
92
|
+
- - "<"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '8'
|
61
95
|
- !ruby/object:Gem::Dependency
|
62
96
|
name: rake
|
63
97
|
requirement: !ruby/object:Gem::Requirement
|
64
98
|
requirements:
|
65
99
|
- - "~>"
|
66
100
|
- !ruby/object:Gem::Version
|
67
|
-
version: '
|
101
|
+
version: '13.0'
|
68
102
|
type: :development
|
69
103
|
prerelease: false
|
70
104
|
version_requirements: !ruby/object:Gem::Requirement
|
71
105
|
requirements:
|
72
106
|
- - "~>"
|
73
107
|
- !ruby/object:Gem::Version
|
74
|
-
version: '
|
108
|
+
version: '13.0'
|
75
109
|
- !ruby/object:Gem::Dependency
|
76
110
|
name: rspec
|
77
111
|
requirement: !ruby/object:Gem::Requirement
|
78
112
|
requirements:
|
79
113
|
- - ">="
|
80
114
|
- !ruby/object:Gem::Version
|
81
|
-
version: '3.
|
115
|
+
version: '3.8'
|
82
116
|
type: :development
|
83
117
|
prerelease: false
|
84
118
|
version_requirements: !ruby/object:Gem::Requirement
|
85
119
|
requirements:
|
86
120
|
- - ">="
|
87
121
|
- !ruby/object:Gem::Version
|
88
|
-
version: '3.
|
122
|
+
version: '3.8'
|
89
123
|
- !ruby/object:Gem::Dependency
|
90
|
-
name:
|
124
|
+
name: rspec_junit_formatter
|
91
125
|
requirement: !ruby/object:Gem::Requirement
|
92
126
|
requirements:
|
93
127
|
- - ">="
|
94
128
|
- !ruby/object:Gem::Version
|
95
|
-
version: '
|
129
|
+
version: '0'
|
96
130
|
type: :development
|
97
131
|
prerelease: false
|
98
132
|
version_requirements: !ruby/object:Gem::Requirement
|
99
133
|
requirements:
|
100
134
|
- - ">="
|
101
135
|
- !ruby/object:Gem::Version
|
102
|
-
version: '
|
136
|
+
version: '0'
|
103
137
|
- !ruby/object:Gem::Dependency
|
104
|
-
name:
|
138
|
+
name: salsify_rubocop
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - "~>"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: 1.27.1
|
144
|
+
type: :development
|
145
|
+
prerelease: false
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - "~>"
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: 1.27.1
|
151
|
+
- !ruby/object:Gem::Dependency
|
152
|
+
name: sprockets
|
153
|
+
requirement: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - "<"
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '4'
|
158
|
+
type: :development
|
159
|
+
prerelease: false
|
160
|
+
version_requirements: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - "<"
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '4'
|
165
|
+
- !ruby/object:Gem::Dependency
|
166
|
+
name: sqlite3
|
105
167
|
requirement: !ruby/object:Gem::Requirement
|
106
168
|
requirements:
|
107
169
|
- - ">="
|
108
170
|
- !ruby/object:Gem::Version
|
109
|
-
version: '
|
171
|
+
version: '1.3'
|
110
172
|
type: :development
|
111
173
|
prerelease: false
|
112
174
|
version_requirements: !ruby/object:Gem::Requirement
|
113
175
|
requirements:
|
114
176
|
- - ">="
|
115
177
|
- !ruby/object:Gem::Version
|
116
|
-
version: '
|
178
|
+
version: '1.3'
|
117
179
|
description:
|
118
180
|
email:
|
119
181
|
- jturkel@salsify.com
|
@@ -122,8 +184,11 @@ executables:
|
|
122
184
|
extensions: []
|
123
185
|
extra_rdoc_files: []
|
124
186
|
files:
|
187
|
+
- ".circleci/config.yml"
|
188
|
+
- ".github/CODEOWNERS"
|
125
189
|
- ".gitignore"
|
126
|
-
- ".
|
190
|
+
- ".rubocop.yml"
|
191
|
+
- Appraisals
|
127
192
|
- CHANGELOG.md
|
128
193
|
- Gemfile
|
129
194
|
- LICENSE.txt
|
@@ -131,17 +196,24 @@ files:
|
|
131
196
|
- Rakefile
|
132
197
|
- bin/delayed_job_worker_pool
|
133
198
|
- delayed_job_worker_pool.gemspec
|
199
|
+
- gemfiles/rails_6.0.gemfile
|
200
|
+
- gemfiles/rails_6.1.gemfile
|
201
|
+
- gemfiles/rails_7.0.gemfile
|
134
202
|
- lib/delayed_job_worker_pool.rb
|
135
203
|
- lib/delayed_job_worker_pool/application.rb
|
136
204
|
- lib/delayed_job_worker_pool/dsl.rb
|
205
|
+
- lib/delayed_job_worker_pool/registry.rb
|
137
206
|
- lib/delayed_job_worker_pool/version.rb
|
138
207
|
- lib/delayed_job_worker_pool/worker.rb
|
208
|
+
- lib/delayed_job_worker_pool/worker_group_options.rb
|
139
209
|
- lib/delayed_job_worker_pool/worker_info.rb
|
140
210
|
- lib/delayed_job_worker_pool/worker_pool.rb
|
141
211
|
homepage: https://github.com/salsify/delayed_job_worker_pool
|
142
212
|
licenses:
|
143
213
|
- MIT
|
144
|
-
metadata:
|
214
|
+
metadata:
|
215
|
+
allowed_push_host: https://rubygems.org
|
216
|
+
rubygems_mfa_required: 'true'
|
145
217
|
post_install_message:
|
146
218
|
rdoc_options: []
|
147
219
|
require_paths:
|
@@ -150,15 +222,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
150
222
|
requirements:
|
151
223
|
- - ">="
|
152
224
|
- !ruby/object:Gem::Version
|
153
|
-
version: '2.
|
225
|
+
version: '2.7'
|
154
226
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
155
227
|
requirements:
|
156
228
|
- - ">="
|
157
229
|
- !ruby/object:Gem::Version
|
158
230
|
version: '0'
|
159
231
|
requirements: []
|
160
|
-
|
161
|
-
rubygems_version: 2.4.7
|
232
|
+
rubygems_version: 3.3.7
|
162
233
|
signing_key:
|
163
234
|
specification_version: 4
|
164
235
|
summary: Worker process pooling for Delayed Job
|