delayed_job_worker_pool 0.2.2 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 1bac50138eccfeaa9cdf6ba18d9e07670081b5c1
4
- data.tar.gz: 3ba969f9f8efe1b488f7ac00d73c94690839e821
2
+ SHA256:
3
+ metadata.gz: ec3712fc2bae4eaac3d8d07ab408616c2efef0763d0d173ff65281d267a42f30
4
+ data.tar.gz: cf6f8866ad4de83cd52e110d5f8d69578af11fa3ba6397448b32400ba711a048
5
5
  SHA512:
6
- metadata.gz: 1d9e7470aa1f9c5c3b57af2b8b26f8201395f9871766860b5ef6dd1d7e81d2427a8cfeb0702f3e250b9a048883b918c9c1a7c968d804b9caafd4f130cd7b2233
7
- data.tar.gz: 8c54d43c3d6b4ed7c017ea5c98e561b5d4a7278b20c9c99e3ca6c626384339407665a3bb0cdfae3f89cc0921fa75f477b2c24684b9d8c106a4f2298b06bcf61c
6
+ metadata.gz: 5e095fcb89fa76967608d5d4e513cce17034d209d24f03b7531a39d25843a1e8bc80923a2b0a5de66d4e804c0912af853622edde7e441da19b9a52c84b991266
7
+ data.tar.gz: c1fe489f6a6172a3d61f7ad27fd384576cda69cbc0160a49bcf393df813d245e18614ba3321d14d18d5284318f18751db220be193deb2712713d2517722756de
@@ -0,0 +1,80 @@
1
+ version: 2.1
2
+ jobs:
3
+ lint:
4
+ docker:
5
+ - image: salsify/ruby_ci:2.7.6
6
+ working_directory: ~/delayed_job_worker_pool
7
+ steps:
8
+ - checkout
9
+ - restore_cache:
10
+ keys:
11
+ - v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
12
+ - v1-gems-ruby-2.7.6-
13
+ - run:
14
+ name: Install Gems
15
+ command: |
16
+ if ! bundle check --path=vendor/bundle; then
17
+ bundle install --path=vendor/bundle --jobs=4 --retry=3
18
+ bundle clean
19
+ fi
20
+ - save_cache:
21
+ key: v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
22
+ paths:
23
+ - "vendor/bundle"
24
+ - "gemfiles/vendor/bundle"
25
+ - run:
26
+ name: Run Rubocop
27
+ command: bundle exec rubocop --config .rubocop.yml
28
+ test:
29
+ parameters:
30
+ ruby_version:
31
+ type: string
32
+ gemfile:
33
+ type: string
34
+ docker:
35
+ - image: salsify/ruby_ci:<< parameters.ruby_version >>
36
+ environment:
37
+ CIRCLE_TEST_REPORTS: "test-results"
38
+ BUNDLE_GEMFILE: "~/delayed_job_worker_pool/<< parameters.gemfile >>"
39
+ working_directory: ~/delayed_job_worker_pool
40
+ steps:
41
+ - checkout
42
+ - restore_cache:
43
+ keys:
44
+ - v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
45
+ - v1-gems-ruby-<< parameters.ruby_version >>-
46
+ - run:
47
+ name: Install Gems
48
+ command: |
49
+ if ! bundle check --path=vendor/bundle; then
50
+ bundle install --path=vendor/bundle --jobs=4 --retry=3
51
+ bundle clean
52
+ fi
53
+ - save_cache:
54
+ key: v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
55
+ paths:
56
+ - "vendor/bundle"
57
+ - "gemfiles/vendor/bundle"
58
+ - run:
59
+ name: Run Tests
60
+ command: |
61
+ bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
62
+ - store_test_results:
63
+ path: "test-results"
64
+ - store_artifacts:
65
+ path: "tmp/log/"
66
+ workflows:
67
+ build:
68
+ jobs:
69
+ - lint
70
+ - test:
71
+ matrix:
72
+ parameters:
73
+ gemfile:
74
+ - "gemfiles/rails_6.0.gemfile"
75
+ - "gemfiles/rails_6.1.gemfile"
76
+ - "gemfiles/rails_7.0.gemfile"
77
+ ruby_version:
78
+ - "2.7.6"
79
+ - "3.0.4"
80
+ - "3.1.2"
@@ -0,0 +1 @@
1
+ * @jturkel @will89
data/.gitignore CHANGED
@@ -9,3 +9,6 @@
9
9
  /tmp/
10
10
  .rspec
11
11
  *.sqlite3
12
+ gemfiles/*.gemfile.lock
13
+ gemfiles/.bundle/*
14
+ spec/dummy/log/
data/.rubocop.yml ADDED
@@ -0,0 +1,8 @@
1
+ inherit_gem:
2
+ salsify_rubocop: conf/rubocop.yml
3
+
4
+ AllCops:
5
+ TargetRubyVersion: 2.7
6
+ Exclude:
7
+ - 'vendor/**/*'
8
+ - 'gemfiles/**/*'
data/Appraisals ADDED
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ appraise 'rails-6.0' do
4
+ gem 'rails', '~> 6.0.5'
5
+ end
6
+
7
+ appraise 'rails-6.1' do
8
+ gem 'rails', '~> 6.1.6'
9
+ end
10
+
11
+ appraise 'rails-7.0' do
12
+ gem 'rails', '~> 7.0.3'
13
+ end
data/CHANGELOG.md CHANGED
@@ -1,6 +1,18 @@
1
1
  # Changelog
2
2
 
3
- ### 0.2.2 (unreleased)
3
+ ### 1.0.0
4
+ * Require Ruby 2.7 or higher.
5
+
6
+ ### 0.3.0
7
+ * Require Ruby 2.5 or higher.
8
+ * Support for running multiple worker pools on a single node.
9
+ See [#7](https://github.com/salsify/delayed_job_worker_pool/pull/7) for details.
10
+ Thanks to Severin Räz!
11
+
12
+ ### 0.2.3
13
+ * Explicitly require 'fcntl' to fix uninitialized constant IO::Fcntl. Thanks to Stefan Wrobel!
14
+
15
+ ### 0.2.2
4
16
  * Add support for Delayed Job 4.1
5
17
 
6
18
  ### 0.2.1
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
5
  gemspec
data/README.md CHANGED
@@ -41,9 +41,11 @@ delayed_job_worker_pool <config file>
41
41
  The config file is a Ruby DSL inspired by the [Puma](https://github.com/puma/puma) configuration DSL. Here's an example:
42
42
 
43
43
  ```ruby
44
- workers Integer(ENV['NUM_WORKERS'] || 1)
45
- queues (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
46
- sleep_delay ENV['WORKER_SLEEP_DELAY']
44
+ worker_group do |g|
45
+ g.workers = Integer(ENV['NUM_WORKERS'] || 1)
46
+ g.queues = (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
47
+ g.sleep_delay = ENV['WORKER_SLEEP_DELAY']
48
+ end
47
49
 
48
50
  preload_app
49
51
 
@@ -77,10 +79,26 @@ after_worker_shutdown do |worker_info|
77
79
  end
78
80
  ```
79
81
 
82
+ You can configure multiple worker groups, i.e.:
83
+
84
+ ```
85
+ worker_group(:default) do |g|
86
+ g.workers = 1
87
+ g.queues = ['default']
88
+ end
89
+
90
+ worker_group(:mails) do |g|
91
+ g.workers = 1
92
+ g.queues = ['mail']
93
+ end
94
+
95
+ ```
96
+
80
97
  Here's more information on each setting:
81
98
 
82
- * `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
83
- * Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
99
+ * `worker_group` - You need at least one worker group. Group settings can be set as illustrated above. Worker group settings:
100
+ * `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
101
+ * Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
84
102
  * `preload_app` - This forces the master process to load Rails before forking worker processes causing the memory consumed by the code to be shared between workers. **If you use this setting make sure you re-establish any necessary connections in the on_worker_boot callback.**
85
103
  * `after_preload_app` - A callback that runs in the master process after preloading the app but before forking any workers.
86
104
  * `on_worker_boot` - A callback that runs in the worker process after it has been forked.
@@ -89,11 +107,15 @@ Here's more information on each setting:
89
107
 
90
108
  All settings are optional and nil values are ignored.
91
109
 
110
+ ## Upgrading from v0.2.x
111
+
112
+ * Convert your worker settings to a single worker group (see _Usage_)
113
+ * Please note the delayed job worker names changed to include ` group: <group_name>`, e.g. if you are monitoring them by their name
114
+
92
115
  ## Contributing
93
116
 
94
117
  Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_worker_pool.
95
118
 
96
-
97
119
  ## License
98
120
 
99
121
  The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'bundler/gem_tasks'
2
4
  require 'rspec/core/rake_task'
3
5
 
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
4
  unless ARGV.size == 1
4
5
  puts "usage: #{File.basename(__FILE__)} <config>"
@@ -1,5 +1,6 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
3
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
5
  require 'delayed_job_worker_pool/version'
5
6
 
@@ -13,18 +14,29 @@ Gem::Specification.new do |spec|
13
14
  spec.homepage = 'https://github.com/salsify/delayed_job_worker_pool'
14
15
  spec.license = 'MIT'
15
16
 
17
+ if spec.respond_to?(:metadata)
18
+ spec.metadata['allowed_push_host'] = 'https://rubygems.org'
19
+ spec.metadata['rubygems_mfa_required'] = 'true'
20
+ else
21
+ raise 'RubyGems 2.0 or newer is required to set allowed_push_host.'
22
+ end
23
+
16
24
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
25
  spec.executables = ['delayed_job_worker_pool']
18
26
  spec.require_paths = ['lib']
19
27
 
20
- spec.required_ruby_version = '>= 2.0'
28
+ spec.required_ruby_version = '>= 2.7'
21
29
 
22
30
  spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.2']
23
31
 
32
+ spec.add_development_dependency 'appraisal'
33
+ spec.add_development_dependency 'bundler', '~> 2.0'
24
34
  spec.add_development_dependency 'delayed_job_active_record'
25
- spec.add_development_dependency 'bundler', '~> 1.10'
26
- spec.add_development_dependency 'rake', '~> 10.0'
27
- spec.add_development_dependency 'rspec', '>= 3.3'
35
+ spec.add_development_dependency 'rails', '>= 6.0', '< 8'
36
+ spec.add_development_dependency 'rake', '~> 13.0'
37
+ spec.add_development_dependency 'rspec', '>= 3.8'
38
+ spec.add_development_dependency 'rspec_junit_formatter'
39
+ spec.add_development_dependency 'salsify_rubocop', '~> 1.27.1'
40
+ spec.add_development_dependency 'sprockets', '< 4'
28
41
  spec.add_development_dependency 'sqlite3', '>= 1.3'
29
- spec.add_development_dependency 'rails', '>= 4.2'
30
42
  end
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 6.0.5"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 6.1.6"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 7.0.3"
6
+
7
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  module Application
3
5
  extend self
@@ -6,7 +8,7 @@ module DelayedJobWorkerPool
6
8
  require(base_application_filename)
7
9
  rescue LoadError
8
10
  raise "Could not find Rails initialization file #{full_application_filename}. " \
9
- "Make sure delayed_job_worker_pool is run from the Rails root directory."
11
+ 'Make sure delayed_job_worker_pool is run from the Rails root directory.'
10
12
  end
11
13
 
12
14
  private
@@ -1,29 +1,47 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  class DSL
3
- SIMPLE_SETTINGS = [:workers, :queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
5
+ class NoWorkerGroupsDefined < StandardError; end
6
+ class NonUniqueGroupName < StandardError; end
7
+
4
8
  CALLBACK_SETTINGS = [:after_preload_app, :on_worker_boot, :after_worker_boot, :after_worker_shutdown].freeze
9
+ DEFAULT_WORKER_GROUP_NAME = :default
5
10
 
6
11
  def self.load(path)
7
12
  options = {}
8
13
 
9
14
  dsl = new(options)
10
15
  dsl.instance_eval(File.read(path), path, 1)
16
+ dsl.assert_groups_defined!
11
17
 
12
18
  options
13
19
  end
14
20
 
15
21
  def initialize(options)
16
22
  @options = options
23
+ @options[:worker_groups] ||= {}
24
+ end
25
+
26
+ def preload_app(preload = true)
27
+ @options[:preload_app] = preload
17
28
  end
18
29
 
19
- SIMPLE_SETTINGS.each do |option_name|
20
- define_method(option_name) do |option_value|
21
- @options[option_name] = option_value unless option_value.nil?
30
+ def worker_group(name = DEFAULT_WORKER_GROUP_NAME)
31
+ name_sym = name.to_sym
32
+ if @options[:worker_groups].key?(name_sym)
33
+ raise NonUniqueGroupName.new("Worker group name #{name_sym} is already in use")
22
34
  end
35
+
36
+ group_options = WorkerGroupOptions.new
37
+ yield(group_options)
38
+ @options[:worker_groups][name_sym] = group_options
23
39
  end
24
40
 
25
- def preload_app(preload_app = true)
26
- @options[:preload_app] = preload_app
41
+ def assert_groups_defined!
42
+ return unless @options[:worker_groups].empty?
43
+
44
+ raise NoWorkerGroupsDefined.new('No worker groups defined. Define groups using `worker_group`.')
27
45
  end
28
46
 
29
47
  CALLBACK_SETTINGS.each do |option_name|
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DelayedJobWorkerPool
4
+ # Keeps track of worker groups and their workers.
5
+ class Registry
6
+ class GroupAlreadyExists < StandardError; end
7
+ class GroupDoesNotExist < StandardError; end
8
+ class GroupNotFound < StandardError; end
9
+
10
+ def initialize
11
+ @groups = {}
12
+ end
13
+
14
+ def include_worker?(pid)
15
+ worker_pids.include?(pid)
16
+ end
17
+
18
+ def workers?
19
+ !worker_pids.empty?
20
+ end
21
+
22
+ def add_group(name, options)
23
+ raise GroupAlreadyExists.new("Group #{group} already exists") if @groups.key?(name)
24
+
25
+ @groups[name] = {
26
+ options: options,
27
+ pids: []
28
+ }
29
+ end
30
+
31
+ def add_worker(group_name, pid)
32
+ group_by_name(group_name)[:pids] << pid
33
+ end
34
+
35
+ def remove_worker(pid)
36
+ @groups[group(pid)][:pids].delete(pid)
37
+ end
38
+
39
+ def options(group_name)
40
+ group_by_name(group_name)[:options]
41
+ end
42
+
43
+ def worker_pids
44
+ @groups.values.flat_map { |v| v[:pids] }
45
+ end
46
+
47
+ def group(pid)
48
+ @groups.each do |name, group|
49
+ return name if group[:pids].include?(pid)
50
+ end
51
+ raise GroupNotFound.new("No group found for PID #{pid}")
52
+ end
53
+
54
+ private
55
+
56
+ def group_by_name(name)
57
+ match = @groups[name]
58
+ return match unless match.nil?
59
+
60
+ raise GroupDoesNotExist.new("No group with name #{name.inspect} found")
61
+ end
62
+ end
63
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
- VERSION = '0.2.2'.freeze
4
+ VERSION = '1.0.0'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  module Worker
3
5
  extend self
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DelayedJobWorkerPool
4
+ class WorkerGroupOptions
5
+ DJ_SETTINGS = [:queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
6
+ GROUP_SETTINGS = [:workers].freeze
7
+
8
+ attr_accessor *DJ_SETTINGS, *GROUP_SETTINGS
9
+
10
+ # @return an options hash for `Delayed::Worker`
11
+ def dj_worker_options
12
+ DJ_SETTINGS.each_with_object({}) do |setting, memo|
13
+ memo[setting] = send(setting)
14
+ end.compact
15
+ end
16
+ end
17
+ end
@@ -1,11 +1,13 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  class WorkerInfo
3
- attr_reader :process_id, :name
5
+ attr_reader :process_id, :name, :worker_group
4
6
 
5
7
  def initialize(attributes)
6
8
  @process_id = attributes.fetch(:process_id)
7
9
  @name = attributes.fetch(:name)
10
+ @worker_group = attributes.fetch(:worker_group)
8
11
  end
9
-
10
12
  end
11
13
  end
@@ -1,10 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fcntl'
4
+ require 'socket'
5
+
1
6
  module DelayedJobWorkerPool
2
7
  class WorkerPool
8
+
3
9
  SIGNALS = ['TERM', 'INT'].map(&:freeze).freeze
10
+ DEFAULT_WORKER_COUNT = 1
4
11
 
5
12
  def initialize(options = {})
6
13
  @options = options
7
- @worker_pids = []
14
+ @registry = Registry.new
8
15
  @pending_signals = []
9
16
  @pending_signal_read_pipe, @pending_signal_write_pipe = create_pipe(inheritable: false)
10
17
  @master_alive_read_pipe, @master_alive_write_pipe = create_pipe(inheritable: true)
@@ -23,7 +30,7 @@ module DelayedJobWorkerPool
23
30
 
24
31
  log_uninheritable_threads
25
32
 
26
- num_workers.times { fork_worker }
33
+ fork_workers
27
34
 
28
35
  monitor_workers
29
36
 
@@ -35,7 +42,7 @@ module DelayedJobWorkerPool
35
42
 
36
43
  private
37
44
 
38
- attr_reader :options, :worker_pids, :master_alive_read_pipe, :master_alive_write_pipe,
45
+ attr_reader :options, :registry, :master_alive_read_pipe, :master_alive_write_pipe,
39
46
  :pending_signals, :pending_signal_read_pipe, :pending_signal_write_pipe
40
47
  attr_accessor :shutting_down
41
48
 
@@ -57,8 +64,10 @@ module DelayedJobWorkerPool
57
64
  def log_uninheritable_threads
58
65
  Thread.list.each do |t|
59
66
  next if t == Thread.current
67
+
60
68
  if t.respond_to?(:backtrace)
61
- log("WARNING: Thread will not be inherited by workers: #{t.inspect} - #{t.backtrace ? t.backtrace.first : ''}")
69
+ log("WARNING: Thread will not be inherited by workers: #{t.inspect} - " \
70
+ "#{t.backtrace ? t.backtrace.first : ''}")
62
71
  else
63
72
  log("WARNING: Thread will not be inherited by workers: #{t.inspect}")
64
73
  end
@@ -72,15 +81,16 @@ module DelayedJobWorkerPool
72
81
  def shutdown(signal)
73
82
  log("Shutting down master #{Process.pid} with signal #{signal}")
74
83
  self.shutting_down = true
75
- worker_pids.each do |child_pid|
76
- log("Telling worker #{child_pid} to shutdown with signal #{signal}")
84
+ registry.worker_pids.each do |child_pid|
85
+ group = registry.group(child_pid)
86
+ log("Telling worker #{child_pid} from group #{group} to shutdown with signal #{signal}")
77
87
  Process.kill(signal, child_pid)
78
88
  end
79
89
  end
80
90
 
81
91
  def monitor_workers
82
- while has_workers?
83
- if has_pending_signal?
92
+ while workers?
93
+ if pending_signal?
84
94
  shutdown(pending_signals.pop)
85
95
  elsif (wait_result = Process.wait2(-1, Process::WNOHANG))
86
96
  handle_dead_worker(wait_result.first, wait_result.last)
@@ -91,19 +101,22 @@ module DelayedJobWorkerPool
91
101
  end
92
102
 
93
103
  def handle_dead_worker(worker_pid, status)
94
- return unless worker_pids.include?(worker_pid)
104
+ return unless registry.include_worker?(worker_pid)
95
105
 
96
106
  log("Worker #{worker_pid} exited with status #{status.to_i}")
97
- worker_pids.delete(worker_pid)
98
- invoke_callback(:after_worker_shutdown, worker_info(worker_pid))
99
- fork_worker unless shutting_down
107
+
108
+ group = registry.group(worker_pid)
109
+ invoke_callback(:after_worker_shutdown, worker_info(worker_pid, group))
110
+
111
+ registry.remove_worker(worker_pid)
112
+ fork_worker(group) unless shutting_down
100
113
  end
101
114
 
102
- def has_workers?
103
- !worker_pids.empty?
115
+ def workers?
116
+ registry.workers?
104
117
  end
105
118
 
106
- def has_pending_signal?
119
+ def pending_signal?
107
120
  !pending_signals.empty?
108
121
  end
109
122
 
@@ -111,14 +124,26 @@ module DelayedJobWorkerPool
111
124
  options[callback_name].call(*args) if options[callback_name]
112
125
  end
113
126
 
114
- def fork_worker
115
- worker_pid = Kernel.fork { run_worker }
116
- worker_pids << worker_pid
117
- log("Started worker #{worker_pid}")
118
- invoke_callback(:after_worker_boot, worker_info(worker_pid))
127
+ def fork_workers
128
+ options.fetch(:worker_groups).each do |name, group|
129
+ workers = group.workers || DEFAULT_WORKER_COUNT
130
+
131
+ registry.add_group(name, group.dj_worker_options)
132
+
133
+ workers.times { fork_worker(name) }
134
+ end
135
+ end
136
+
137
+ def fork_worker(group)
138
+ worker_pid = Kernel.fork { run_worker(group) }
139
+ log("Started worker in group #{group}: #{worker_pid}")
140
+
141
+ registry.add_worker(group, worker_pid)
142
+
143
+ invoke_callback(:after_worker_boot, worker_info(worker_pid, group))
119
144
  end
120
145
 
121
- def run_worker
146
+ def run_worker(group)
122
147
  master_alive_write_pipe.close
123
148
 
124
149
  uninstall_signal_handlers
@@ -131,32 +156,32 @@ module DelayedJobWorkerPool
131
156
 
132
157
  load_app unless preload_app?
133
158
 
134
- invoke_callback(:on_worker_boot, worker_info(Process.pid))
159
+ invoke_callback(:on_worker_boot, worker_info(Process.pid, group))
135
160
 
136
- DelayedJobWorkerPool::Worker.run(worker_options(Process.pid))
137
- rescue => e
161
+ DelayedJobWorkerPool::Worker.run(worker_options(Process.pid, group))
162
+ rescue StandardError => e
138
163
  log("Worker failed with error: #{e.message}\n#{e.backtrace.join("\n")}")
139
164
  exit(1)
140
165
  end
141
166
 
142
- def worker_info(worker_pid)
143
- DelayedJobWorkerPool::WorkerInfo.new(name: worker_name(worker_pid), process_id: worker_pid)
144
- end
145
-
146
- def worker_name(worker_pid)
147
- "host:#{Socket.gethostname} pid:#{worker_pid}"
167
+ def worker_info(worker_pid, group)
168
+ DelayedJobWorkerPool::WorkerInfo.new(
169
+ name: worker_name(worker_pid, group),
170
+ process_id: worker_pid,
171
+ worker_group: group
172
+ )
148
173
  end
149
174
 
150
- def num_workers
151
- options.fetch(:workers, 1)
175
+ def worker_name(worker_pid, group)
176
+ "host:#{Socket.gethostname} pid:#{worker_pid} group:#{group}"
152
177
  end
153
178
 
154
179
  def preload_app?
155
180
  options.fetch(:preload_app, false)
156
181
  end
157
182
 
158
- def worker_options(worker_pid)
159
- options.except(:workers, :preload_app, *DelayedJobWorkerPool::DSL::CALLBACK_SETTINGS).merge(name: worker_name(worker_pid))
183
+ def worker_options(worker_pid, group)
184
+ registry.options(group).merge(name: worker_name(worker_pid, group))
160
185
  end
161
186
 
162
187
  def create_pipe(inheritable: true)
@@ -173,9 +198,7 @@ module DelayedJobWorkerPool
173
198
  end
174
199
 
175
200
  def wait_for_signal(timeout)
176
- if IO.select([pending_signal_read_pipe], [], [], timeout)
177
- drain_pipe(pending_signal_read_pipe)
178
- end
201
+ drain_pipe(pending_signal_read_pipe) if IO.select([pending_signal_read_pipe], [], [], timeout)
179
202
  end
180
203
 
181
204
  def drain_pipe(pipe)
@@ -1,6 +1,10 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'delayed_job_worker_pool/application'
2
4
  require 'delayed_job_worker_pool/dsl'
5
+ require 'delayed_job_worker_pool/registry'
3
6
  require 'delayed_job_worker_pool/worker'
4
7
  require 'delayed_job_worker_pool/worker_info'
5
8
  require 'delayed_job_worker_pool/worker_pool'
9
+ require 'delayed_job_worker_pool/worker_group_options'
6
10
  require 'delayed_job_worker_pool/version'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: delayed_job_worker_pool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joel Turkel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-06 00:00:00.000000000 Z
11
+ date: 2022-09-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: delayed_job
@@ -31,7 +31,7 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '4.2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: delayed_job_active_record
34
+ name: appraisal
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - ">="
@@ -50,70 +50,132 @@ dependencies:
50
50
  requirements:
51
51
  - - "~>"
52
52
  - !ruby/object:Gem::Version
53
- version: '1.10'
53
+ version: '2.0'
54
54
  type: :development
55
55
  prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
58
  - - "~>"
59
59
  - !ruby/object:Gem::Version
60
- version: '1.10'
60
+ version: '2.0'
61
+ - !ruby/object:Gem::Dependency
62
+ name: delayed_job_active_record
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: rails
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '6.0'
82
+ - - "<"
83
+ - !ruby/object:Gem::Version
84
+ version: '8'
85
+ type: :development
86
+ prerelease: false
87
+ version_requirements: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '6.0'
92
+ - - "<"
93
+ - !ruby/object:Gem::Version
94
+ version: '8'
61
95
  - !ruby/object:Gem::Dependency
62
96
  name: rake
63
97
  requirement: !ruby/object:Gem::Requirement
64
98
  requirements:
65
99
  - - "~>"
66
100
  - !ruby/object:Gem::Version
67
- version: '10.0'
101
+ version: '13.0'
68
102
  type: :development
69
103
  prerelease: false
70
104
  version_requirements: !ruby/object:Gem::Requirement
71
105
  requirements:
72
106
  - - "~>"
73
107
  - !ruby/object:Gem::Version
74
- version: '10.0'
108
+ version: '13.0'
75
109
  - !ruby/object:Gem::Dependency
76
110
  name: rspec
77
111
  requirement: !ruby/object:Gem::Requirement
78
112
  requirements:
79
113
  - - ">="
80
114
  - !ruby/object:Gem::Version
81
- version: '3.3'
115
+ version: '3.8'
82
116
  type: :development
83
117
  prerelease: false
84
118
  version_requirements: !ruby/object:Gem::Requirement
85
119
  requirements:
86
120
  - - ">="
87
121
  - !ruby/object:Gem::Version
88
- version: '3.3'
122
+ version: '3.8'
89
123
  - !ruby/object:Gem::Dependency
90
- name: sqlite3
124
+ name: rspec_junit_formatter
91
125
  requirement: !ruby/object:Gem::Requirement
92
126
  requirements:
93
127
  - - ">="
94
128
  - !ruby/object:Gem::Version
95
- version: '1.3'
129
+ version: '0'
96
130
  type: :development
97
131
  prerelease: false
98
132
  version_requirements: !ruby/object:Gem::Requirement
99
133
  requirements:
100
134
  - - ">="
101
135
  - !ruby/object:Gem::Version
102
- version: '1.3'
136
+ version: '0'
103
137
  - !ruby/object:Gem::Dependency
104
- name: rails
138
+ name: salsify_rubocop
139
+ requirement: !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - "~>"
142
+ - !ruby/object:Gem::Version
143
+ version: 1.27.1
144
+ type: :development
145
+ prerelease: false
146
+ version_requirements: !ruby/object:Gem::Requirement
147
+ requirements:
148
+ - - "~>"
149
+ - !ruby/object:Gem::Version
150
+ version: 1.27.1
151
+ - !ruby/object:Gem::Dependency
152
+ name: sprockets
153
+ requirement: !ruby/object:Gem::Requirement
154
+ requirements:
155
+ - - "<"
156
+ - !ruby/object:Gem::Version
157
+ version: '4'
158
+ type: :development
159
+ prerelease: false
160
+ version_requirements: !ruby/object:Gem::Requirement
161
+ requirements:
162
+ - - "<"
163
+ - !ruby/object:Gem::Version
164
+ version: '4'
165
+ - !ruby/object:Gem::Dependency
166
+ name: sqlite3
105
167
  requirement: !ruby/object:Gem::Requirement
106
168
  requirements:
107
169
  - - ">="
108
170
  - !ruby/object:Gem::Version
109
- version: '4.2'
171
+ version: '1.3'
110
172
  type: :development
111
173
  prerelease: false
112
174
  version_requirements: !ruby/object:Gem::Requirement
113
175
  requirements:
114
176
  - - ">="
115
177
  - !ruby/object:Gem::Version
116
- version: '4.2'
178
+ version: '1.3'
117
179
  description:
118
180
  email:
119
181
  - jturkel@salsify.com
@@ -122,8 +184,11 @@ executables:
122
184
  extensions: []
123
185
  extra_rdoc_files: []
124
186
  files:
187
+ - ".circleci/config.yml"
188
+ - ".github/CODEOWNERS"
125
189
  - ".gitignore"
126
- - ".travis.yml"
190
+ - ".rubocop.yml"
191
+ - Appraisals
127
192
  - CHANGELOG.md
128
193
  - Gemfile
129
194
  - LICENSE.txt
@@ -131,17 +196,24 @@ files:
131
196
  - Rakefile
132
197
  - bin/delayed_job_worker_pool
133
198
  - delayed_job_worker_pool.gemspec
199
+ - gemfiles/rails_6.0.gemfile
200
+ - gemfiles/rails_6.1.gemfile
201
+ - gemfiles/rails_7.0.gemfile
134
202
  - lib/delayed_job_worker_pool.rb
135
203
  - lib/delayed_job_worker_pool/application.rb
136
204
  - lib/delayed_job_worker_pool/dsl.rb
205
+ - lib/delayed_job_worker_pool/registry.rb
137
206
  - lib/delayed_job_worker_pool/version.rb
138
207
  - lib/delayed_job_worker_pool/worker.rb
208
+ - lib/delayed_job_worker_pool/worker_group_options.rb
139
209
  - lib/delayed_job_worker_pool/worker_info.rb
140
210
  - lib/delayed_job_worker_pool/worker_pool.rb
141
211
  homepage: https://github.com/salsify/delayed_job_worker_pool
142
212
  licenses:
143
213
  - MIT
144
- metadata: {}
214
+ metadata:
215
+ allowed_push_host: https://rubygems.org
216
+ rubygems_mfa_required: 'true'
145
217
  post_install_message:
146
218
  rdoc_options: []
147
219
  require_paths:
@@ -150,15 +222,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
150
222
  requirements:
151
223
  - - ">="
152
224
  - !ruby/object:Gem::Version
153
- version: '2.0'
225
+ version: '2.7'
154
226
  required_rubygems_version: !ruby/object:Gem::Requirement
155
227
  requirements:
156
228
  - - ">="
157
229
  - !ruby/object:Gem::Version
158
230
  version: '0'
159
231
  requirements: []
160
- rubyforge_project:
161
- rubygems_version: 2.4.7
232
+ rubygems_version: 3.3.7
162
233
  signing_key:
163
234
  specification_version: 4
164
235
  summary: Worker process pooling for Delayed Job
data/.travis.yml DELETED
@@ -1,6 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.2.3
4
- - 2.1.7
5
- - 2.0.0-p647
6
- before_install: gem install bundler -v 1.10.6