delayed_job_worker_pool 0.2.2 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +80 -0
- data/.github/CODEOWNERS +1 -0
- data/.gitignore +3 -0
- data/.rubocop.yml +8 -0
- data/Appraisals +13 -0
- data/CHANGELOG.md +13 -1
- data/Gemfile +2 -0
- data/README.md +28 -6
- data/Rakefile +2 -0
- data/bin/delayed_job_worker_pool +1 -0
- data/delayed_job_worker_pool.gemspec +19 -7
- data/gemfiles/rails_6.0.gemfile +7 -0
- data/gemfiles/rails_6.1.gemfile +7 -0
- data/gemfiles/rails_7.0.gemfile +7 -0
- data/lib/delayed_job_worker_pool/application.rb +3 -1
- data/lib/delayed_job_worker_pool/dsl.rb +24 -6
- data/lib/delayed_job_worker_pool/registry.rb +63 -0
- data/lib/delayed_job_worker_pool/version.rb +3 -1
- data/lib/delayed_job_worker_pool/worker.rb +2 -0
- data/lib/delayed_job_worker_pool/worker_group_options.rb +17 -0
- data/lib/delayed_job_worker_pool/worker_info.rb +4 -2
- data/lib/delayed_job_worker_pool/worker_pool.rb +60 -37
- data/lib/delayed_job_worker_pool.rb +4 -0
- metadata +91 -20
- data/.travis.yml +0 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: ec3712fc2bae4eaac3d8d07ab408616c2efef0763d0d173ff65281d267a42f30
|
4
|
+
data.tar.gz: cf6f8866ad4de83cd52e110d5f8d69578af11fa3ba6397448b32400ba711a048
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5e095fcb89fa76967608d5d4e513cce17034d209d24f03b7531a39d25843a1e8bc80923a2b0a5de66d4e804c0912af853622edde7e441da19b9a52c84b991266
|
7
|
+
data.tar.gz: c1fe489f6a6172a3d61f7ad27fd384576cda69cbc0160a49bcf393df813d245e18614ba3321d14d18d5284318f18751db220be193deb2712713d2517722756de
|
@@ -0,0 +1,80 @@
|
|
1
|
+
version: 2.1
|
2
|
+
jobs:
|
3
|
+
lint:
|
4
|
+
docker:
|
5
|
+
- image: salsify/ruby_ci:2.7.6
|
6
|
+
working_directory: ~/delayed_job_worker_pool
|
7
|
+
steps:
|
8
|
+
- checkout
|
9
|
+
- restore_cache:
|
10
|
+
keys:
|
11
|
+
- v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
|
12
|
+
- v1-gems-ruby-2.7.6-
|
13
|
+
- run:
|
14
|
+
name: Install Gems
|
15
|
+
command: |
|
16
|
+
if ! bundle check --path=vendor/bundle; then
|
17
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
18
|
+
bundle clean
|
19
|
+
fi
|
20
|
+
- save_cache:
|
21
|
+
key: v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
|
22
|
+
paths:
|
23
|
+
- "vendor/bundle"
|
24
|
+
- "gemfiles/vendor/bundle"
|
25
|
+
- run:
|
26
|
+
name: Run Rubocop
|
27
|
+
command: bundle exec rubocop --config .rubocop.yml
|
28
|
+
test:
|
29
|
+
parameters:
|
30
|
+
ruby_version:
|
31
|
+
type: string
|
32
|
+
gemfile:
|
33
|
+
type: string
|
34
|
+
docker:
|
35
|
+
- image: salsify/ruby_ci:<< parameters.ruby_version >>
|
36
|
+
environment:
|
37
|
+
CIRCLE_TEST_REPORTS: "test-results"
|
38
|
+
BUNDLE_GEMFILE: "~/delayed_job_worker_pool/<< parameters.gemfile >>"
|
39
|
+
working_directory: ~/delayed_job_worker_pool
|
40
|
+
steps:
|
41
|
+
- checkout
|
42
|
+
- restore_cache:
|
43
|
+
keys:
|
44
|
+
- v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
45
|
+
- v1-gems-ruby-<< parameters.ruby_version >>-
|
46
|
+
- run:
|
47
|
+
name: Install Gems
|
48
|
+
command: |
|
49
|
+
if ! bundle check --path=vendor/bundle; then
|
50
|
+
bundle install --path=vendor/bundle --jobs=4 --retry=3
|
51
|
+
bundle clean
|
52
|
+
fi
|
53
|
+
- save_cache:
|
54
|
+
key: v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
|
55
|
+
paths:
|
56
|
+
- "vendor/bundle"
|
57
|
+
- "gemfiles/vendor/bundle"
|
58
|
+
- run:
|
59
|
+
name: Run Tests
|
60
|
+
command: |
|
61
|
+
bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
|
62
|
+
- store_test_results:
|
63
|
+
path: "test-results"
|
64
|
+
- store_artifacts:
|
65
|
+
path: "tmp/log/"
|
66
|
+
workflows:
|
67
|
+
build:
|
68
|
+
jobs:
|
69
|
+
- lint
|
70
|
+
- test:
|
71
|
+
matrix:
|
72
|
+
parameters:
|
73
|
+
gemfile:
|
74
|
+
- "gemfiles/rails_6.0.gemfile"
|
75
|
+
- "gemfiles/rails_6.1.gemfile"
|
76
|
+
- "gemfiles/rails_7.0.gemfile"
|
77
|
+
ruby_version:
|
78
|
+
- "2.7.6"
|
79
|
+
- "3.0.4"
|
80
|
+
- "3.1.2"
|
data/.github/CODEOWNERS
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* @jturkel @will89
|
data/.gitignore
CHANGED
data/.rubocop.yml
ADDED
data/Appraisals
ADDED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,18 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
### 0.
|
3
|
+
### 1.0.0
|
4
|
+
* Require Ruby 2.7 or higher.
|
5
|
+
|
6
|
+
### 0.3.0
|
7
|
+
* Require Ruby 2.5 or higher.
|
8
|
+
* Support for running multiple worker pools on a single node.
|
9
|
+
See [#7](https://github.com/salsify/delayed_job_worker_pool/pull/7) for details.
|
10
|
+
Thanks to Severin Räz!
|
11
|
+
|
12
|
+
### 0.2.3
|
13
|
+
* Explicitly require 'fcntl' to fix uninitialized constant IO::Fcntl. Thanks to Stefan Wrobel!
|
14
|
+
|
15
|
+
### 0.2.2
|
4
16
|
* Add support for Delayed Job 4.1
|
5
17
|
|
6
18
|
### 0.2.1
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -41,9 +41,11 @@ delayed_job_worker_pool <config file>
|
|
41
41
|
The config file is a Ruby DSL inspired by the [Puma](https://github.com/puma/puma) configuration DSL. Here's an example:
|
42
42
|
|
43
43
|
```ruby
|
44
|
-
|
45
|
-
|
46
|
-
|
44
|
+
worker_group do |g|
|
45
|
+
g.workers = Integer(ENV['NUM_WORKERS'] || 1)
|
46
|
+
g.queues = (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
|
47
|
+
g.sleep_delay = ENV['WORKER_SLEEP_DELAY']
|
48
|
+
end
|
47
49
|
|
48
50
|
preload_app
|
49
51
|
|
@@ -77,10 +79,26 @@ after_worker_shutdown do |worker_info|
|
|
77
79
|
end
|
78
80
|
```
|
79
81
|
|
82
|
+
You can configure multiple worker groups, i.e.:
|
83
|
+
|
84
|
+
```
|
85
|
+
worker_group(:default) do |g|
|
86
|
+
g.workers = 1
|
87
|
+
g.queues = ['default']
|
88
|
+
end
|
89
|
+
|
90
|
+
worker_group(:mails) do |g|
|
91
|
+
g.workers = 1
|
92
|
+
g.queues = ['mail']
|
93
|
+
end
|
94
|
+
|
95
|
+
```
|
96
|
+
|
80
97
|
Here's more information on each setting:
|
81
98
|
|
82
|
-
* `
|
83
|
-
*
|
99
|
+
* `worker_group` - You need at least one worker group. Group settings can be set as illustrated above. Worker group settings:
|
100
|
+
* `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
|
101
|
+
* Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
|
84
102
|
* `preload_app` - This forces the master process to load Rails before forking worker processes causing the memory consumed by the code to be shared between workers. **If you use this setting make sure you re-establish any necessary connections in the on_worker_boot callback.**
|
85
103
|
* `after_preload_app` - A callback that runs in the master process after preloading the app but before forking any workers.
|
86
104
|
* `on_worker_boot` - A callback that runs in the worker process after it has been forked.
|
@@ -89,11 +107,15 @@ Here's more information on each setting:
|
|
89
107
|
|
90
108
|
All settings are optional and nil values are ignored.
|
91
109
|
|
110
|
+
## Upgrading from v0.2.x
|
111
|
+
|
112
|
+
* Convert your worker settings to a single worker group (see _Usage_)
|
113
|
+
* Please note the delayed job worker names changed to include ` group: <group_name>`, e.g. if you are monitoring them by their name
|
114
|
+
|
92
115
|
## Contributing
|
93
116
|
|
94
117
|
Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_worker_pool.
|
95
118
|
|
96
|
-
|
97
119
|
## License
|
98
120
|
|
99
121
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/Rakefile
CHANGED
data/bin/delayed_job_worker_pool
CHANGED
@@ -1,5 +1,6 @@
|
|
1
|
-
#
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
lib = File.expand_path('lib', __dir__)
|
3
4
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
5
|
require 'delayed_job_worker_pool/version'
|
5
6
|
|
@@ -13,18 +14,29 @@ Gem::Specification.new do |spec|
|
|
13
14
|
spec.homepage = 'https://github.com/salsify/delayed_job_worker_pool'
|
14
15
|
spec.license = 'MIT'
|
15
16
|
|
17
|
+
if spec.respond_to?(:metadata)
|
18
|
+
spec.metadata['allowed_push_host'] = 'https://rubygems.org'
|
19
|
+
spec.metadata['rubygems_mfa_required'] = 'true'
|
20
|
+
else
|
21
|
+
raise 'RubyGems 2.0 or newer is required to set allowed_push_host.'
|
22
|
+
end
|
23
|
+
|
16
24
|
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
17
25
|
spec.executables = ['delayed_job_worker_pool']
|
18
26
|
spec.require_paths = ['lib']
|
19
27
|
|
20
|
-
spec.required_ruby_version = '>= 2.
|
28
|
+
spec.required_ruby_version = '>= 2.7'
|
21
29
|
|
22
30
|
spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.2']
|
23
31
|
|
32
|
+
spec.add_development_dependency 'appraisal'
|
33
|
+
spec.add_development_dependency 'bundler', '~> 2.0'
|
24
34
|
spec.add_development_dependency 'delayed_job_active_record'
|
25
|
-
spec.add_development_dependency '
|
26
|
-
spec.add_development_dependency 'rake', '~>
|
27
|
-
spec.add_development_dependency 'rspec', '>= 3.
|
35
|
+
spec.add_development_dependency 'rails', '>= 6.0', '< 8'
|
36
|
+
spec.add_development_dependency 'rake', '~> 13.0'
|
37
|
+
spec.add_development_dependency 'rspec', '>= 3.8'
|
38
|
+
spec.add_development_dependency 'rspec_junit_formatter'
|
39
|
+
spec.add_development_dependency 'salsify_rubocop', '~> 1.27.1'
|
40
|
+
spec.add_development_dependency 'sprockets', '< 4'
|
28
41
|
spec.add_development_dependency 'sqlite3', '>= 1.3'
|
29
|
-
spec.add_development_dependency 'rails', '>= 4.2'
|
30
42
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
module Application
|
3
5
|
extend self
|
@@ -6,7 +8,7 @@ module DelayedJobWorkerPool
|
|
6
8
|
require(base_application_filename)
|
7
9
|
rescue LoadError
|
8
10
|
raise "Could not find Rails initialization file #{full_application_filename}. " \
|
9
|
-
|
11
|
+
'Make sure delayed_job_worker_pool is run from the Rails root directory.'
|
10
12
|
end
|
11
13
|
|
12
14
|
private
|
@@ -1,29 +1,47 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
class DSL
|
3
|
-
|
5
|
+
class NoWorkerGroupsDefined < StandardError; end
|
6
|
+
class NonUniqueGroupName < StandardError; end
|
7
|
+
|
4
8
|
CALLBACK_SETTINGS = [:after_preload_app, :on_worker_boot, :after_worker_boot, :after_worker_shutdown].freeze
|
9
|
+
DEFAULT_WORKER_GROUP_NAME = :default
|
5
10
|
|
6
11
|
def self.load(path)
|
7
12
|
options = {}
|
8
13
|
|
9
14
|
dsl = new(options)
|
10
15
|
dsl.instance_eval(File.read(path), path, 1)
|
16
|
+
dsl.assert_groups_defined!
|
11
17
|
|
12
18
|
options
|
13
19
|
end
|
14
20
|
|
15
21
|
def initialize(options)
|
16
22
|
@options = options
|
23
|
+
@options[:worker_groups] ||= {}
|
24
|
+
end
|
25
|
+
|
26
|
+
def preload_app(preload = true)
|
27
|
+
@options[:preload_app] = preload
|
17
28
|
end
|
18
29
|
|
19
|
-
|
20
|
-
|
21
|
-
|
30
|
+
def worker_group(name = DEFAULT_WORKER_GROUP_NAME)
|
31
|
+
name_sym = name.to_sym
|
32
|
+
if @options[:worker_groups].key?(name_sym)
|
33
|
+
raise NonUniqueGroupName.new("Worker group name #{name_sym} is already in use")
|
22
34
|
end
|
35
|
+
|
36
|
+
group_options = WorkerGroupOptions.new
|
37
|
+
yield(group_options)
|
38
|
+
@options[:worker_groups][name_sym] = group_options
|
23
39
|
end
|
24
40
|
|
25
|
-
def
|
26
|
-
@options[:
|
41
|
+
def assert_groups_defined!
|
42
|
+
return unless @options[:worker_groups].empty?
|
43
|
+
|
44
|
+
raise NoWorkerGroupsDefined.new('No worker groups defined. Define groups using `worker_group`.')
|
27
45
|
end
|
28
46
|
|
29
47
|
CALLBACK_SETTINGS.each do |option_name|
|
@@ -0,0 +1,63 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DelayedJobWorkerPool
|
4
|
+
# Keeps track of worker groups and their workers.
|
5
|
+
class Registry
|
6
|
+
class GroupAlreadyExists < StandardError; end
|
7
|
+
class GroupDoesNotExist < StandardError; end
|
8
|
+
class GroupNotFound < StandardError; end
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@groups = {}
|
12
|
+
end
|
13
|
+
|
14
|
+
def include_worker?(pid)
|
15
|
+
worker_pids.include?(pid)
|
16
|
+
end
|
17
|
+
|
18
|
+
def workers?
|
19
|
+
!worker_pids.empty?
|
20
|
+
end
|
21
|
+
|
22
|
+
def add_group(name, options)
|
23
|
+
raise GroupAlreadyExists.new("Group #{group} already exists") if @groups.key?(name)
|
24
|
+
|
25
|
+
@groups[name] = {
|
26
|
+
options: options,
|
27
|
+
pids: []
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def add_worker(group_name, pid)
|
32
|
+
group_by_name(group_name)[:pids] << pid
|
33
|
+
end
|
34
|
+
|
35
|
+
def remove_worker(pid)
|
36
|
+
@groups[group(pid)][:pids].delete(pid)
|
37
|
+
end
|
38
|
+
|
39
|
+
def options(group_name)
|
40
|
+
group_by_name(group_name)[:options]
|
41
|
+
end
|
42
|
+
|
43
|
+
def worker_pids
|
44
|
+
@groups.values.flat_map { |v| v[:pids] }
|
45
|
+
end
|
46
|
+
|
47
|
+
def group(pid)
|
48
|
+
@groups.each do |name, group|
|
49
|
+
return name if group[:pids].include?(pid)
|
50
|
+
end
|
51
|
+
raise GroupNotFound.new("No group found for PID #{pid}")
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def group_by_name(name)
|
57
|
+
match = @groups[name]
|
58
|
+
return match unless match.nil?
|
59
|
+
|
60
|
+
raise GroupDoesNotExist.new("No group with name #{name.inspect} found")
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module DelayedJobWorkerPool
|
4
|
+
class WorkerGroupOptions
|
5
|
+
DJ_SETTINGS = [:queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
|
6
|
+
GROUP_SETTINGS = [:workers].freeze
|
7
|
+
|
8
|
+
attr_accessor *DJ_SETTINGS, *GROUP_SETTINGS
|
9
|
+
|
10
|
+
# @return an options hash for `Delayed::Worker`
|
11
|
+
def dj_worker_options
|
12
|
+
DJ_SETTINGS.each_with_object({}) do |setting, memo|
|
13
|
+
memo[setting] = send(setting)
|
14
|
+
end.compact
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
@@ -1,11 +1,13 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module DelayedJobWorkerPool
|
2
4
|
class WorkerInfo
|
3
|
-
attr_reader :process_id, :name
|
5
|
+
attr_reader :process_id, :name, :worker_group
|
4
6
|
|
5
7
|
def initialize(attributes)
|
6
8
|
@process_id = attributes.fetch(:process_id)
|
7
9
|
@name = attributes.fetch(:name)
|
10
|
+
@worker_group = attributes.fetch(:worker_group)
|
8
11
|
end
|
9
|
-
|
10
12
|
end
|
11
13
|
end
|
@@ -1,10 +1,17 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'fcntl'
|
4
|
+
require 'socket'
|
5
|
+
|
1
6
|
module DelayedJobWorkerPool
|
2
7
|
class WorkerPool
|
8
|
+
|
3
9
|
SIGNALS = ['TERM', 'INT'].map(&:freeze).freeze
|
10
|
+
DEFAULT_WORKER_COUNT = 1
|
4
11
|
|
5
12
|
def initialize(options = {})
|
6
13
|
@options = options
|
7
|
-
@
|
14
|
+
@registry = Registry.new
|
8
15
|
@pending_signals = []
|
9
16
|
@pending_signal_read_pipe, @pending_signal_write_pipe = create_pipe(inheritable: false)
|
10
17
|
@master_alive_read_pipe, @master_alive_write_pipe = create_pipe(inheritable: true)
|
@@ -23,7 +30,7 @@ module DelayedJobWorkerPool
|
|
23
30
|
|
24
31
|
log_uninheritable_threads
|
25
32
|
|
26
|
-
|
33
|
+
fork_workers
|
27
34
|
|
28
35
|
monitor_workers
|
29
36
|
|
@@ -35,7 +42,7 @@ module DelayedJobWorkerPool
|
|
35
42
|
|
36
43
|
private
|
37
44
|
|
38
|
-
attr_reader :options, :
|
45
|
+
attr_reader :options, :registry, :master_alive_read_pipe, :master_alive_write_pipe,
|
39
46
|
:pending_signals, :pending_signal_read_pipe, :pending_signal_write_pipe
|
40
47
|
attr_accessor :shutting_down
|
41
48
|
|
@@ -57,8 +64,10 @@ module DelayedJobWorkerPool
|
|
57
64
|
def log_uninheritable_threads
|
58
65
|
Thread.list.each do |t|
|
59
66
|
next if t == Thread.current
|
67
|
+
|
60
68
|
if t.respond_to?(:backtrace)
|
61
|
-
log("WARNING: Thread will not be inherited by workers: #{t.inspect} -
|
69
|
+
log("WARNING: Thread will not be inherited by workers: #{t.inspect} - " \
|
70
|
+
"#{t.backtrace ? t.backtrace.first : ''}")
|
62
71
|
else
|
63
72
|
log("WARNING: Thread will not be inherited by workers: #{t.inspect}")
|
64
73
|
end
|
@@ -72,15 +81,16 @@ module DelayedJobWorkerPool
|
|
72
81
|
def shutdown(signal)
|
73
82
|
log("Shutting down master #{Process.pid} with signal #{signal}")
|
74
83
|
self.shutting_down = true
|
75
|
-
worker_pids.each do |child_pid|
|
76
|
-
|
84
|
+
registry.worker_pids.each do |child_pid|
|
85
|
+
group = registry.group(child_pid)
|
86
|
+
log("Telling worker #{child_pid} from group #{group} to shutdown with signal #{signal}")
|
77
87
|
Process.kill(signal, child_pid)
|
78
88
|
end
|
79
89
|
end
|
80
90
|
|
81
91
|
def monitor_workers
|
82
|
-
while
|
83
|
-
if
|
92
|
+
while workers?
|
93
|
+
if pending_signal?
|
84
94
|
shutdown(pending_signals.pop)
|
85
95
|
elsif (wait_result = Process.wait2(-1, Process::WNOHANG))
|
86
96
|
handle_dead_worker(wait_result.first, wait_result.last)
|
@@ -91,19 +101,22 @@ module DelayedJobWorkerPool
|
|
91
101
|
end
|
92
102
|
|
93
103
|
def handle_dead_worker(worker_pid, status)
|
94
|
-
return unless
|
104
|
+
return unless registry.include_worker?(worker_pid)
|
95
105
|
|
96
106
|
log("Worker #{worker_pid} exited with status #{status.to_i}")
|
97
|
-
|
98
|
-
|
99
|
-
|
107
|
+
|
108
|
+
group = registry.group(worker_pid)
|
109
|
+
invoke_callback(:after_worker_shutdown, worker_info(worker_pid, group))
|
110
|
+
|
111
|
+
registry.remove_worker(worker_pid)
|
112
|
+
fork_worker(group) unless shutting_down
|
100
113
|
end
|
101
114
|
|
102
|
-
def
|
103
|
-
|
115
|
+
def workers?
|
116
|
+
registry.workers?
|
104
117
|
end
|
105
118
|
|
106
|
-
def
|
119
|
+
def pending_signal?
|
107
120
|
!pending_signals.empty?
|
108
121
|
end
|
109
122
|
|
@@ -111,14 +124,26 @@ module DelayedJobWorkerPool
|
|
111
124
|
options[callback_name].call(*args) if options[callback_name]
|
112
125
|
end
|
113
126
|
|
114
|
-
def
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
127
|
+
def fork_workers
|
128
|
+
options.fetch(:worker_groups).each do |name, group|
|
129
|
+
workers = group.workers || DEFAULT_WORKER_COUNT
|
130
|
+
|
131
|
+
registry.add_group(name, group.dj_worker_options)
|
132
|
+
|
133
|
+
workers.times { fork_worker(name) }
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def fork_worker(group)
|
138
|
+
worker_pid = Kernel.fork { run_worker(group) }
|
139
|
+
log("Started worker in group #{group}: #{worker_pid}")
|
140
|
+
|
141
|
+
registry.add_worker(group, worker_pid)
|
142
|
+
|
143
|
+
invoke_callback(:after_worker_boot, worker_info(worker_pid, group))
|
119
144
|
end
|
120
145
|
|
121
|
-
def run_worker
|
146
|
+
def run_worker(group)
|
122
147
|
master_alive_write_pipe.close
|
123
148
|
|
124
149
|
uninstall_signal_handlers
|
@@ -131,32 +156,32 @@ module DelayedJobWorkerPool
|
|
131
156
|
|
132
157
|
load_app unless preload_app?
|
133
158
|
|
134
|
-
invoke_callback(:on_worker_boot, worker_info(Process.pid))
|
159
|
+
invoke_callback(:on_worker_boot, worker_info(Process.pid, group))
|
135
160
|
|
136
|
-
DelayedJobWorkerPool::Worker.run(worker_options(Process.pid))
|
137
|
-
rescue => e
|
161
|
+
DelayedJobWorkerPool::Worker.run(worker_options(Process.pid, group))
|
162
|
+
rescue StandardError => e
|
138
163
|
log("Worker failed with error: #{e.message}\n#{e.backtrace.join("\n")}")
|
139
164
|
exit(1)
|
140
165
|
end
|
141
166
|
|
142
|
-
def worker_info(worker_pid)
|
143
|
-
DelayedJobWorkerPool::WorkerInfo.new(
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
167
|
+
def worker_info(worker_pid, group)
|
168
|
+
DelayedJobWorkerPool::WorkerInfo.new(
|
169
|
+
name: worker_name(worker_pid, group),
|
170
|
+
process_id: worker_pid,
|
171
|
+
worker_group: group
|
172
|
+
)
|
148
173
|
end
|
149
174
|
|
150
|
-
def
|
151
|
-
|
175
|
+
def worker_name(worker_pid, group)
|
176
|
+
"host:#{Socket.gethostname} pid:#{worker_pid} group:#{group}"
|
152
177
|
end
|
153
178
|
|
154
179
|
def preload_app?
|
155
180
|
options.fetch(:preload_app, false)
|
156
181
|
end
|
157
182
|
|
158
|
-
def worker_options(worker_pid)
|
159
|
-
options
|
183
|
+
def worker_options(worker_pid, group)
|
184
|
+
registry.options(group).merge(name: worker_name(worker_pid, group))
|
160
185
|
end
|
161
186
|
|
162
187
|
def create_pipe(inheritable: true)
|
@@ -173,9 +198,7 @@ module DelayedJobWorkerPool
|
|
173
198
|
end
|
174
199
|
|
175
200
|
def wait_for_signal(timeout)
|
176
|
-
if IO.select([pending_signal_read_pipe], [], [], timeout)
|
177
|
-
drain_pipe(pending_signal_read_pipe)
|
178
|
-
end
|
201
|
+
drain_pipe(pending_signal_read_pipe) if IO.select([pending_signal_read_pipe], [], [], timeout)
|
179
202
|
end
|
180
203
|
|
181
204
|
def drain_pipe(pipe)
|
@@ -1,6 +1,10 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'delayed_job_worker_pool/application'
|
2
4
|
require 'delayed_job_worker_pool/dsl'
|
5
|
+
require 'delayed_job_worker_pool/registry'
|
3
6
|
require 'delayed_job_worker_pool/worker'
|
4
7
|
require 'delayed_job_worker_pool/worker_info'
|
5
8
|
require 'delayed_job_worker_pool/worker_pool'
|
9
|
+
require 'delayed_job_worker_pool/worker_group_options'
|
6
10
|
require 'delayed_job_worker_pool/version'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: delayed_job_worker_pool
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Joel Turkel
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2022-09-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: delayed_job
|
@@ -31,7 +31,7 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '4.2'
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: appraisal
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
37
|
- - ">="
|
@@ -50,70 +50,132 @@ dependencies:
|
|
50
50
|
requirements:
|
51
51
|
- - "~>"
|
52
52
|
- !ruby/object:Gem::Version
|
53
|
-
version: '
|
53
|
+
version: '2.0'
|
54
54
|
type: :development
|
55
55
|
prerelease: false
|
56
56
|
version_requirements: !ruby/object:Gem::Requirement
|
57
57
|
requirements:
|
58
58
|
- - "~>"
|
59
59
|
- !ruby/object:Gem::Version
|
60
|
-
version: '
|
60
|
+
version: '2.0'
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: delayed_job_active_record
|
63
|
+
requirement: !ruby/object:Gem::Requirement
|
64
|
+
requirements:
|
65
|
+
- - ">="
|
66
|
+
- !ruby/object:Gem::Version
|
67
|
+
version: '0'
|
68
|
+
type: :development
|
69
|
+
prerelease: false
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
name: rails
|
77
|
+
requirement: !ruby/object:Gem::Requirement
|
78
|
+
requirements:
|
79
|
+
- - ">="
|
80
|
+
- !ruby/object:Gem::Version
|
81
|
+
version: '6.0'
|
82
|
+
- - "<"
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: '8'
|
85
|
+
type: :development
|
86
|
+
prerelease: false
|
87
|
+
version_requirements: !ruby/object:Gem::Requirement
|
88
|
+
requirements:
|
89
|
+
- - ">="
|
90
|
+
- !ruby/object:Gem::Version
|
91
|
+
version: '6.0'
|
92
|
+
- - "<"
|
93
|
+
- !ruby/object:Gem::Version
|
94
|
+
version: '8'
|
61
95
|
- !ruby/object:Gem::Dependency
|
62
96
|
name: rake
|
63
97
|
requirement: !ruby/object:Gem::Requirement
|
64
98
|
requirements:
|
65
99
|
- - "~>"
|
66
100
|
- !ruby/object:Gem::Version
|
67
|
-
version: '
|
101
|
+
version: '13.0'
|
68
102
|
type: :development
|
69
103
|
prerelease: false
|
70
104
|
version_requirements: !ruby/object:Gem::Requirement
|
71
105
|
requirements:
|
72
106
|
- - "~>"
|
73
107
|
- !ruby/object:Gem::Version
|
74
|
-
version: '
|
108
|
+
version: '13.0'
|
75
109
|
- !ruby/object:Gem::Dependency
|
76
110
|
name: rspec
|
77
111
|
requirement: !ruby/object:Gem::Requirement
|
78
112
|
requirements:
|
79
113
|
- - ">="
|
80
114
|
- !ruby/object:Gem::Version
|
81
|
-
version: '3.
|
115
|
+
version: '3.8'
|
82
116
|
type: :development
|
83
117
|
prerelease: false
|
84
118
|
version_requirements: !ruby/object:Gem::Requirement
|
85
119
|
requirements:
|
86
120
|
- - ">="
|
87
121
|
- !ruby/object:Gem::Version
|
88
|
-
version: '3.
|
122
|
+
version: '3.8'
|
89
123
|
- !ruby/object:Gem::Dependency
|
90
|
-
name:
|
124
|
+
name: rspec_junit_formatter
|
91
125
|
requirement: !ruby/object:Gem::Requirement
|
92
126
|
requirements:
|
93
127
|
- - ">="
|
94
128
|
- !ruby/object:Gem::Version
|
95
|
-
version: '
|
129
|
+
version: '0'
|
96
130
|
type: :development
|
97
131
|
prerelease: false
|
98
132
|
version_requirements: !ruby/object:Gem::Requirement
|
99
133
|
requirements:
|
100
134
|
- - ">="
|
101
135
|
- !ruby/object:Gem::Version
|
102
|
-
version: '
|
136
|
+
version: '0'
|
103
137
|
- !ruby/object:Gem::Dependency
|
104
|
-
name:
|
138
|
+
name: salsify_rubocop
|
139
|
+
requirement: !ruby/object:Gem::Requirement
|
140
|
+
requirements:
|
141
|
+
- - "~>"
|
142
|
+
- !ruby/object:Gem::Version
|
143
|
+
version: 1.27.1
|
144
|
+
type: :development
|
145
|
+
prerelease: false
|
146
|
+
version_requirements: !ruby/object:Gem::Requirement
|
147
|
+
requirements:
|
148
|
+
- - "~>"
|
149
|
+
- !ruby/object:Gem::Version
|
150
|
+
version: 1.27.1
|
151
|
+
- !ruby/object:Gem::Dependency
|
152
|
+
name: sprockets
|
153
|
+
requirement: !ruby/object:Gem::Requirement
|
154
|
+
requirements:
|
155
|
+
- - "<"
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '4'
|
158
|
+
type: :development
|
159
|
+
prerelease: false
|
160
|
+
version_requirements: !ruby/object:Gem::Requirement
|
161
|
+
requirements:
|
162
|
+
- - "<"
|
163
|
+
- !ruby/object:Gem::Version
|
164
|
+
version: '4'
|
165
|
+
- !ruby/object:Gem::Dependency
|
166
|
+
name: sqlite3
|
105
167
|
requirement: !ruby/object:Gem::Requirement
|
106
168
|
requirements:
|
107
169
|
- - ">="
|
108
170
|
- !ruby/object:Gem::Version
|
109
|
-
version: '
|
171
|
+
version: '1.3'
|
110
172
|
type: :development
|
111
173
|
prerelease: false
|
112
174
|
version_requirements: !ruby/object:Gem::Requirement
|
113
175
|
requirements:
|
114
176
|
- - ">="
|
115
177
|
- !ruby/object:Gem::Version
|
116
|
-
version: '
|
178
|
+
version: '1.3'
|
117
179
|
description:
|
118
180
|
email:
|
119
181
|
- jturkel@salsify.com
|
@@ -122,8 +184,11 @@ executables:
|
|
122
184
|
extensions: []
|
123
185
|
extra_rdoc_files: []
|
124
186
|
files:
|
187
|
+
- ".circleci/config.yml"
|
188
|
+
- ".github/CODEOWNERS"
|
125
189
|
- ".gitignore"
|
126
|
-
- ".
|
190
|
+
- ".rubocop.yml"
|
191
|
+
- Appraisals
|
127
192
|
- CHANGELOG.md
|
128
193
|
- Gemfile
|
129
194
|
- LICENSE.txt
|
@@ -131,17 +196,24 @@ files:
|
|
131
196
|
- Rakefile
|
132
197
|
- bin/delayed_job_worker_pool
|
133
198
|
- delayed_job_worker_pool.gemspec
|
199
|
+
- gemfiles/rails_6.0.gemfile
|
200
|
+
- gemfiles/rails_6.1.gemfile
|
201
|
+
- gemfiles/rails_7.0.gemfile
|
134
202
|
- lib/delayed_job_worker_pool.rb
|
135
203
|
- lib/delayed_job_worker_pool/application.rb
|
136
204
|
- lib/delayed_job_worker_pool/dsl.rb
|
205
|
+
- lib/delayed_job_worker_pool/registry.rb
|
137
206
|
- lib/delayed_job_worker_pool/version.rb
|
138
207
|
- lib/delayed_job_worker_pool/worker.rb
|
208
|
+
- lib/delayed_job_worker_pool/worker_group_options.rb
|
139
209
|
- lib/delayed_job_worker_pool/worker_info.rb
|
140
210
|
- lib/delayed_job_worker_pool/worker_pool.rb
|
141
211
|
homepage: https://github.com/salsify/delayed_job_worker_pool
|
142
212
|
licenses:
|
143
213
|
- MIT
|
144
|
-
metadata:
|
214
|
+
metadata:
|
215
|
+
allowed_push_host: https://rubygems.org
|
216
|
+
rubygems_mfa_required: 'true'
|
145
217
|
post_install_message:
|
146
218
|
rdoc_options: []
|
147
219
|
require_paths:
|
@@ -150,15 +222,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
150
222
|
requirements:
|
151
223
|
- - ">="
|
152
224
|
- !ruby/object:Gem::Version
|
153
|
-
version: '2.
|
225
|
+
version: '2.7'
|
154
226
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
155
227
|
requirements:
|
156
228
|
- - ">="
|
157
229
|
- !ruby/object:Gem::Version
|
158
230
|
version: '0'
|
159
231
|
requirements: []
|
160
|
-
|
161
|
-
rubygems_version: 2.4.7
|
232
|
+
rubygems_version: 3.3.7
|
162
233
|
signing_key:
|
163
234
|
specification_version: 4
|
164
235
|
summary: Worker process pooling for Delayed Job
|