delayed_job_worker_pool 0.2.2 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: 1bac50138eccfeaa9cdf6ba18d9e07670081b5c1
4
- data.tar.gz: 3ba969f9f8efe1b488f7ac00d73c94690839e821
2
+ SHA256:
3
+ metadata.gz: ec3712fc2bae4eaac3d8d07ab408616c2efef0763d0d173ff65281d267a42f30
4
+ data.tar.gz: cf6f8866ad4de83cd52e110d5f8d69578af11fa3ba6397448b32400ba711a048
5
5
  SHA512:
6
- metadata.gz: 1d9e7470aa1f9c5c3b57af2b8b26f8201395f9871766860b5ef6dd1d7e81d2427a8cfeb0702f3e250b9a048883b918c9c1a7c968d804b9caafd4f130cd7b2233
7
- data.tar.gz: 8c54d43c3d6b4ed7c017ea5c98e561b5d4a7278b20c9c99e3ca6c626384339407665a3bb0cdfae3f89cc0921fa75f477b2c24684b9d8c106a4f2298b06bcf61c
6
+ metadata.gz: 5e095fcb89fa76967608d5d4e513cce17034d209d24f03b7531a39d25843a1e8bc80923a2b0a5de66d4e804c0912af853622edde7e441da19b9a52c84b991266
7
+ data.tar.gz: c1fe489f6a6172a3d61f7ad27fd384576cda69cbc0160a49bcf393df813d245e18614ba3321d14d18d5284318f18751db220be193deb2712713d2517722756de
@@ -0,0 +1,80 @@
1
+ version: 2.1
2
+ jobs:
3
+ lint:
4
+ docker:
5
+ - image: salsify/ruby_ci:2.7.6
6
+ working_directory: ~/delayed_job_worker_pool
7
+ steps:
8
+ - checkout
9
+ - restore_cache:
10
+ keys:
11
+ - v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
12
+ - v1-gems-ruby-2.7.6-
13
+ - run:
14
+ name: Install Gems
15
+ command: |
16
+ if ! bundle check --path=vendor/bundle; then
17
+ bundle install --path=vendor/bundle --jobs=4 --retry=3
18
+ bundle clean
19
+ fi
20
+ - save_cache:
21
+ key: v1-gems-ruby-2.7.6-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "Gemfile" }}
22
+ paths:
23
+ - "vendor/bundle"
24
+ - "gemfiles/vendor/bundle"
25
+ - run:
26
+ name: Run Rubocop
27
+ command: bundle exec rubocop --config .rubocop.yml
28
+ test:
29
+ parameters:
30
+ ruby_version:
31
+ type: string
32
+ gemfile:
33
+ type: string
34
+ docker:
35
+ - image: salsify/ruby_ci:<< parameters.ruby_version >>
36
+ environment:
37
+ CIRCLE_TEST_REPORTS: "test-results"
38
+ BUNDLE_GEMFILE: "~/delayed_job_worker_pool/<< parameters.gemfile >>"
39
+ working_directory: ~/delayed_job_worker_pool
40
+ steps:
41
+ - checkout
42
+ - restore_cache:
43
+ keys:
44
+ - v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
45
+ - v1-gems-ruby-<< parameters.ruby_version >>-
46
+ - run:
47
+ name: Install Gems
48
+ command: |
49
+ if ! bundle check --path=vendor/bundle; then
50
+ bundle install --path=vendor/bundle --jobs=4 --retry=3
51
+ bundle clean
52
+ fi
53
+ - save_cache:
54
+ key: v1-gems-ruby-<< parameters.ruby_version >>-{{ checksum "delayed_job_worker_pool.gemspec" }}-{{ checksum "<< parameters.gemfile >>" }}
55
+ paths:
56
+ - "vendor/bundle"
57
+ - "gemfiles/vendor/bundle"
58
+ - run:
59
+ name: Run Tests
60
+ command: |
61
+ bundle exec rspec --format RspecJunitFormatter --out $CIRCLE_TEST_REPORTS/rspec/junit.xml --format progress spec
62
+ - store_test_results:
63
+ path: "test-results"
64
+ - store_artifacts:
65
+ path: "tmp/log/"
66
+ workflows:
67
+ build:
68
+ jobs:
69
+ - lint
70
+ - test:
71
+ matrix:
72
+ parameters:
73
+ gemfile:
74
+ - "gemfiles/rails_6.0.gemfile"
75
+ - "gemfiles/rails_6.1.gemfile"
76
+ - "gemfiles/rails_7.0.gemfile"
77
+ ruby_version:
78
+ - "2.7.6"
79
+ - "3.0.4"
80
+ - "3.1.2"
@@ -0,0 +1 @@
1
+ * @jturkel @will89
data/.gitignore CHANGED
@@ -9,3 +9,6 @@
9
9
  /tmp/
10
10
  .rspec
11
11
  *.sqlite3
12
+ gemfiles/*.gemfile.lock
13
+ gemfiles/.bundle/*
14
+ spec/dummy/log/
data/.rubocop.yml ADDED
@@ -0,0 +1,8 @@
1
+ inherit_gem:
2
+ salsify_rubocop: conf/rubocop.yml
3
+
4
+ AllCops:
5
+ TargetRubyVersion: 2.7
6
+ Exclude:
7
+ - 'vendor/**/*'
8
+ - 'gemfiles/**/*'
data/Appraisals ADDED
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ appraise 'rails-6.0' do
4
+ gem 'rails', '~> 6.0.5'
5
+ end
6
+
7
+ appraise 'rails-6.1' do
8
+ gem 'rails', '~> 6.1.6'
9
+ end
10
+
11
+ appraise 'rails-7.0' do
12
+ gem 'rails', '~> 7.0.3'
13
+ end
data/CHANGELOG.md CHANGED
@@ -1,6 +1,18 @@
1
1
  # Changelog
2
2
 
3
- ### 0.2.2 (unreleased)
3
+ ### 1.0.0
4
+ * Require Ruby 2.7 or higher.
5
+
6
+ ### 0.3.0
7
+ * Require Ruby 2.5 or higher.
8
+ * Support for running multiple worker pools on a single node.
9
+ See [#7](https://github.com/salsify/delayed_job_worker_pool/pull/7) for details.
10
+ Thanks to Severin Räz!
11
+
12
+ ### 0.2.3
13
+ * Explicitly require 'fcntl' to fix uninitialized constant IO::Fcntl. Thanks to Stefan Wrobel!
14
+
15
+ ### 0.2.2
4
16
  * Add support for Delayed Job 4.1
5
17
 
6
18
  ### 0.2.1
data/Gemfile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  source 'https://rubygems.org'
2
4
 
3
5
  gemspec
data/README.md CHANGED
@@ -41,9 +41,11 @@ delayed_job_worker_pool <config file>
41
41
  The config file is a Ruby DSL inspired by the [Puma](https://github.com/puma/puma) configuration DSL. Here's an example:
42
42
 
43
43
  ```ruby
44
- workers Integer(ENV['NUM_WORKERS'] || 1)
45
- queues (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
46
- sleep_delay ENV['WORKER_SLEEP_DELAY']
44
+ worker_group do |g|
45
+ g.workers = Integer(ENV['NUM_WORKERS'] || 1)
46
+ g.queues = (ENV['QUEUES'] || ENV['QUEUE'] || '').split(',')
47
+ g.sleep_delay = ENV['WORKER_SLEEP_DELAY']
48
+ end
47
49
 
48
50
  preload_app
49
51
 
@@ -77,10 +79,26 @@ after_worker_shutdown do |worker_info|
77
79
  end
78
80
  ```
79
81
 
82
+ You can configure multiple worker groups, i.e.:
83
+
84
+ ```
85
+ worker_group(:default) do |g|
86
+ g.workers = 1
87
+ g.queues = ['default']
88
+ end
89
+
90
+ worker_group(:mails) do |g|
91
+ g.workers = 1
92
+ g.queues = ['mail']
93
+ end
94
+
95
+ ```
96
+
80
97
  Here's more information on each setting:
81
98
 
82
- * `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
83
- * Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
99
+ * `worker_group` - You need at least one worker group. Group settings can be set as illustrated above. Worker group settings:
100
+ * `workers` - The number of Delayed Job worker processes to fork. The master process will relaunch workers that fail.
101
+ * Delayed Job worker settings (`queues`, `min_priority`, `max_priority`, `sleep_delay`, `read_ahead`) - These are passed through to the Delayed Job worker.
84
102
  * `preload_app` - This forces the master process to load Rails before forking worker processes causing the memory consumed by the code to be shared between workers. **If you use this setting make sure you re-establish any necessary connections in the on_worker_boot callback.**
85
103
  * `after_preload_app` - A callback that runs in the master process after preloading the app but before forking any workers.
86
104
  * `on_worker_boot` - A callback that runs in the worker process after it has been forked.
@@ -89,11 +107,15 @@ Here's more information on each setting:
89
107
 
90
108
  All settings are optional and nil values are ignored.
91
109
 
110
+ ## Upgrading from v0.2.x
111
+
112
+ * Convert your worker settings to a single worker group (see _Usage_)
113
+ * Please note the delayed job worker names changed to include ` group: <group_name>`, e.g. if you are monitoring them by their name
114
+
92
115
  ## Contributing
93
116
 
94
117
  Bug reports and pull requests are welcome on GitHub at https://github.com/salsify/delayed_job_worker_pool.
95
118
 
96
-
97
119
  ## License
98
120
 
99
121
  The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
data/Rakefile CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'bundler/gem_tasks'
2
4
  require 'rspec/core/rake_task'
3
5
 
@@ -1,4 +1,5 @@
1
1
  #!/usr/bin/env ruby
2
+ # frozen_string_literal: true
2
3
 
3
4
  unless ARGV.size == 1
4
5
  puts "usage: #{File.basename(__FILE__)} <config>"
@@ -1,5 +1,6 @@
1
- # coding: utf-8
2
- lib = File.expand_path('../lib', __FILE__)
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
3
4
  $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
5
  require 'delayed_job_worker_pool/version'
5
6
 
@@ -13,18 +14,29 @@ Gem::Specification.new do |spec|
13
14
  spec.homepage = 'https://github.com/salsify/delayed_job_worker_pool'
14
15
  spec.license = 'MIT'
15
16
 
17
+ if spec.respond_to?(:metadata)
18
+ spec.metadata['allowed_push_host'] = 'https://rubygems.org'
19
+ spec.metadata['rubygems_mfa_required'] = 'true'
20
+ else
21
+ raise 'RubyGems 2.0 or newer is required to set allowed_push_host.'
22
+ end
23
+
16
24
  spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
25
  spec.executables = ['delayed_job_worker_pool']
18
26
  spec.require_paths = ['lib']
19
27
 
20
- spec.required_ruby_version = '>= 2.0'
28
+ spec.required_ruby_version = '>= 2.7'
21
29
 
22
30
  spec.add_dependency 'delayed_job', ['>= 3.0', '< 4.2']
23
31
 
32
+ spec.add_development_dependency 'appraisal'
33
+ spec.add_development_dependency 'bundler', '~> 2.0'
24
34
  spec.add_development_dependency 'delayed_job_active_record'
25
- spec.add_development_dependency 'bundler', '~> 1.10'
26
- spec.add_development_dependency 'rake', '~> 10.0'
27
- spec.add_development_dependency 'rspec', '>= 3.3'
35
+ spec.add_development_dependency 'rails', '>= 6.0', '< 8'
36
+ spec.add_development_dependency 'rake', '~> 13.0'
37
+ spec.add_development_dependency 'rspec', '>= 3.8'
38
+ spec.add_development_dependency 'rspec_junit_formatter'
39
+ spec.add_development_dependency 'salsify_rubocop', '~> 1.27.1'
40
+ spec.add_development_dependency 'sprockets', '< 4'
28
41
  spec.add_development_dependency 'sqlite3', '>= 1.3'
29
- spec.add_development_dependency 'rails', '>= 4.2'
30
42
  end
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 6.0.5"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 6.1.6"
6
+
7
+ gemspec path: "../"
@@ -0,0 +1,7 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "rails", "~> 7.0.3"
6
+
7
+ gemspec path: "../"
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  module Application
3
5
  extend self
@@ -6,7 +8,7 @@ module DelayedJobWorkerPool
6
8
  require(base_application_filename)
7
9
  rescue LoadError
8
10
  raise "Could not find Rails initialization file #{full_application_filename}. " \
9
- "Make sure delayed_job_worker_pool is run from the Rails root directory."
11
+ 'Make sure delayed_job_worker_pool is run from the Rails root directory.'
10
12
  end
11
13
 
12
14
  private
@@ -1,29 +1,47 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  class DSL
3
- SIMPLE_SETTINGS = [:workers, :queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
5
+ class NoWorkerGroupsDefined < StandardError; end
6
+ class NonUniqueGroupName < StandardError; end
7
+
4
8
  CALLBACK_SETTINGS = [:after_preload_app, :on_worker_boot, :after_worker_boot, :after_worker_shutdown].freeze
9
+ DEFAULT_WORKER_GROUP_NAME = :default
5
10
 
6
11
  def self.load(path)
7
12
  options = {}
8
13
 
9
14
  dsl = new(options)
10
15
  dsl.instance_eval(File.read(path), path, 1)
16
+ dsl.assert_groups_defined!
11
17
 
12
18
  options
13
19
  end
14
20
 
15
21
  def initialize(options)
16
22
  @options = options
23
+ @options[:worker_groups] ||= {}
24
+ end
25
+
26
+ def preload_app(preload = true)
27
+ @options[:preload_app] = preload
17
28
  end
18
29
 
19
- SIMPLE_SETTINGS.each do |option_name|
20
- define_method(option_name) do |option_value|
21
- @options[option_name] = option_value unless option_value.nil?
30
+ def worker_group(name = DEFAULT_WORKER_GROUP_NAME)
31
+ name_sym = name.to_sym
32
+ if @options[:worker_groups].key?(name_sym)
33
+ raise NonUniqueGroupName.new("Worker group name #{name_sym} is already in use")
22
34
  end
35
+
36
+ group_options = WorkerGroupOptions.new
37
+ yield(group_options)
38
+ @options[:worker_groups][name_sym] = group_options
23
39
  end
24
40
 
25
- def preload_app(preload_app = true)
26
- @options[:preload_app] = preload_app
41
+ def assert_groups_defined!
42
+ return unless @options[:worker_groups].empty?
43
+
44
+ raise NoWorkerGroupsDefined.new('No worker groups defined. Define groups using `worker_group`.')
27
45
  end
28
46
 
29
47
  CALLBACK_SETTINGS.each do |option_name|
@@ -0,0 +1,63 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DelayedJobWorkerPool
4
+ # Keeps track of worker groups and their workers.
5
+ class Registry
6
+ class GroupAlreadyExists < StandardError; end
7
+ class GroupDoesNotExist < StandardError; end
8
+ class GroupNotFound < StandardError; end
9
+
10
+ def initialize
11
+ @groups = {}
12
+ end
13
+
14
+ def include_worker?(pid)
15
+ worker_pids.include?(pid)
16
+ end
17
+
18
+ def workers?
19
+ !worker_pids.empty?
20
+ end
21
+
22
+ def add_group(name, options)
23
+ raise GroupAlreadyExists.new("Group #{group} already exists") if @groups.key?(name)
24
+
25
+ @groups[name] = {
26
+ options: options,
27
+ pids: []
28
+ }
29
+ end
30
+
31
+ def add_worker(group_name, pid)
32
+ group_by_name(group_name)[:pids] << pid
33
+ end
34
+
35
+ def remove_worker(pid)
36
+ @groups[group(pid)][:pids].delete(pid)
37
+ end
38
+
39
+ def options(group_name)
40
+ group_by_name(group_name)[:options]
41
+ end
42
+
43
+ def worker_pids
44
+ @groups.values.flat_map { |v| v[:pids] }
45
+ end
46
+
47
+ def group(pid)
48
+ @groups.each do |name, group|
49
+ return name if group[:pids].include?(pid)
50
+ end
51
+ raise GroupNotFound.new("No group found for PID #{pid}")
52
+ end
53
+
54
+ private
55
+
56
+ def group_by_name(name)
57
+ match = @groups[name]
58
+ return match unless match.nil?
59
+
60
+ raise GroupDoesNotExist.new("No group with name #{name.inspect} found")
61
+ end
62
+ end
63
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
- VERSION = '0.2.2'.freeze
4
+ VERSION = '1.0.0'
3
5
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  module Worker
3
5
  extend self
@@ -0,0 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ module DelayedJobWorkerPool
4
+ class WorkerGroupOptions
5
+ DJ_SETTINGS = [:queues, :min_priority, :max_priority, :sleep_delay, :read_ahead].freeze
6
+ GROUP_SETTINGS = [:workers].freeze
7
+
8
+ attr_accessor *DJ_SETTINGS, *GROUP_SETTINGS
9
+
10
+ # @return an options hash for `Delayed::Worker`
11
+ def dj_worker_options
12
+ DJ_SETTINGS.each_with_object({}) do |setting, memo|
13
+ memo[setting] = send(setting)
14
+ end.compact
15
+ end
16
+ end
17
+ end
@@ -1,11 +1,13 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module DelayedJobWorkerPool
2
4
  class WorkerInfo
3
- attr_reader :process_id, :name
5
+ attr_reader :process_id, :name, :worker_group
4
6
 
5
7
  def initialize(attributes)
6
8
  @process_id = attributes.fetch(:process_id)
7
9
  @name = attributes.fetch(:name)
10
+ @worker_group = attributes.fetch(:worker_group)
8
11
  end
9
-
10
12
  end
11
13
  end
@@ -1,10 +1,17 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'fcntl'
4
+ require 'socket'
5
+
1
6
  module DelayedJobWorkerPool
2
7
  class WorkerPool
8
+
3
9
  SIGNALS = ['TERM', 'INT'].map(&:freeze).freeze
10
+ DEFAULT_WORKER_COUNT = 1
4
11
 
5
12
  def initialize(options = {})
6
13
  @options = options
7
- @worker_pids = []
14
+ @registry = Registry.new
8
15
  @pending_signals = []
9
16
  @pending_signal_read_pipe, @pending_signal_write_pipe = create_pipe(inheritable: false)
10
17
  @master_alive_read_pipe, @master_alive_write_pipe = create_pipe(inheritable: true)
@@ -23,7 +30,7 @@ module DelayedJobWorkerPool
23
30
 
24
31
  log_uninheritable_threads
25
32
 
26
- num_workers.times { fork_worker }
33
+ fork_workers
27
34
 
28
35
  monitor_workers
29
36
 
@@ -35,7 +42,7 @@ module DelayedJobWorkerPool
35
42
 
36
43
  private
37
44
 
38
- attr_reader :options, :worker_pids, :master_alive_read_pipe, :master_alive_write_pipe,
45
+ attr_reader :options, :registry, :master_alive_read_pipe, :master_alive_write_pipe,
39
46
  :pending_signals, :pending_signal_read_pipe, :pending_signal_write_pipe
40
47
  attr_accessor :shutting_down
41
48
 
@@ -57,8 +64,10 @@ module DelayedJobWorkerPool
57
64
  def log_uninheritable_threads
58
65
  Thread.list.each do |t|
59
66
  next if t == Thread.current
67
+
60
68
  if t.respond_to?(:backtrace)
61
- log("WARNING: Thread will not be inherited by workers: #{t.inspect} - #{t.backtrace ? t.backtrace.first : ''}")
69
+ log("WARNING: Thread will not be inherited by workers: #{t.inspect} - " \
70
+ "#{t.backtrace ? t.backtrace.first : ''}")
62
71
  else
63
72
  log("WARNING: Thread will not be inherited by workers: #{t.inspect}")
64
73
  end
@@ -72,15 +81,16 @@ module DelayedJobWorkerPool
72
81
  def shutdown(signal)
73
82
  log("Shutting down master #{Process.pid} with signal #{signal}")
74
83
  self.shutting_down = true
75
- worker_pids.each do |child_pid|
76
- log("Telling worker #{child_pid} to shutdown with signal #{signal}")
84
+ registry.worker_pids.each do |child_pid|
85
+ group = registry.group(child_pid)
86
+ log("Telling worker #{child_pid} from group #{group} to shutdown with signal #{signal}")
77
87
  Process.kill(signal, child_pid)
78
88
  end
79
89
  end
80
90
 
81
91
  def monitor_workers
82
- while has_workers?
83
- if has_pending_signal?
92
+ while workers?
93
+ if pending_signal?
84
94
  shutdown(pending_signals.pop)
85
95
  elsif (wait_result = Process.wait2(-1, Process::WNOHANG))
86
96
  handle_dead_worker(wait_result.first, wait_result.last)
@@ -91,19 +101,22 @@ module DelayedJobWorkerPool
91
101
  end
92
102
 
93
103
  def handle_dead_worker(worker_pid, status)
94
- return unless worker_pids.include?(worker_pid)
104
+ return unless registry.include_worker?(worker_pid)
95
105
 
96
106
  log("Worker #{worker_pid} exited with status #{status.to_i}")
97
- worker_pids.delete(worker_pid)
98
- invoke_callback(:after_worker_shutdown, worker_info(worker_pid))
99
- fork_worker unless shutting_down
107
+
108
+ group = registry.group(worker_pid)
109
+ invoke_callback(:after_worker_shutdown, worker_info(worker_pid, group))
110
+
111
+ registry.remove_worker(worker_pid)
112
+ fork_worker(group) unless shutting_down
100
113
  end
101
114
 
102
- def has_workers?
103
- !worker_pids.empty?
115
+ def workers?
116
+ registry.workers?
104
117
  end
105
118
 
106
- def has_pending_signal?
119
+ def pending_signal?
107
120
  !pending_signals.empty?
108
121
  end
109
122
 
@@ -111,14 +124,26 @@ module DelayedJobWorkerPool
111
124
  options[callback_name].call(*args) if options[callback_name]
112
125
  end
113
126
 
114
- def fork_worker
115
- worker_pid = Kernel.fork { run_worker }
116
- worker_pids << worker_pid
117
- log("Started worker #{worker_pid}")
118
- invoke_callback(:after_worker_boot, worker_info(worker_pid))
127
+ def fork_workers
128
+ options.fetch(:worker_groups).each do |name, group|
129
+ workers = group.workers || DEFAULT_WORKER_COUNT
130
+
131
+ registry.add_group(name, group.dj_worker_options)
132
+
133
+ workers.times { fork_worker(name) }
134
+ end
135
+ end
136
+
137
+ def fork_worker(group)
138
+ worker_pid = Kernel.fork { run_worker(group) }
139
+ log("Started worker in group #{group}: #{worker_pid}")
140
+
141
+ registry.add_worker(group, worker_pid)
142
+
143
+ invoke_callback(:after_worker_boot, worker_info(worker_pid, group))
119
144
  end
120
145
 
121
- def run_worker
146
+ def run_worker(group)
122
147
  master_alive_write_pipe.close
123
148
 
124
149
  uninstall_signal_handlers
@@ -131,32 +156,32 @@ module DelayedJobWorkerPool
131
156
 
132
157
  load_app unless preload_app?
133
158
 
134
- invoke_callback(:on_worker_boot, worker_info(Process.pid))
159
+ invoke_callback(:on_worker_boot, worker_info(Process.pid, group))
135
160
 
136
- DelayedJobWorkerPool::Worker.run(worker_options(Process.pid))
137
- rescue => e
161
+ DelayedJobWorkerPool::Worker.run(worker_options(Process.pid, group))
162
+ rescue StandardError => e
138
163
  log("Worker failed with error: #{e.message}\n#{e.backtrace.join("\n")}")
139
164
  exit(1)
140
165
  end
141
166
 
142
- def worker_info(worker_pid)
143
- DelayedJobWorkerPool::WorkerInfo.new(name: worker_name(worker_pid), process_id: worker_pid)
144
- end
145
-
146
- def worker_name(worker_pid)
147
- "host:#{Socket.gethostname} pid:#{worker_pid}"
167
+ def worker_info(worker_pid, group)
168
+ DelayedJobWorkerPool::WorkerInfo.new(
169
+ name: worker_name(worker_pid, group),
170
+ process_id: worker_pid,
171
+ worker_group: group
172
+ )
148
173
  end
149
174
 
150
- def num_workers
151
- options.fetch(:workers, 1)
175
+ def worker_name(worker_pid, group)
176
+ "host:#{Socket.gethostname} pid:#{worker_pid} group:#{group}"
152
177
  end
153
178
 
154
179
  def preload_app?
155
180
  options.fetch(:preload_app, false)
156
181
  end
157
182
 
158
- def worker_options(worker_pid)
159
- options.except(:workers, :preload_app, *DelayedJobWorkerPool::DSL::CALLBACK_SETTINGS).merge(name: worker_name(worker_pid))
183
+ def worker_options(worker_pid, group)
184
+ registry.options(group).merge(name: worker_name(worker_pid, group))
160
185
  end
161
186
 
162
187
  def create_pipe(inheritable: true)
@@ -173,9 +198,7 @@ module DelayedJobWorkerPool
173
198
  end
174
199
 
175
200
  def wait_for_signal(timeout)
176
- if IO.select([pending_signal_read_pipe], [], [], timeout)
177
- drain_pipe(pending_signal_read_pipe)
178
- end
201
+ drain_pipe(pending_signal_read_pipe) if IO.select([pending_signal_read_pipe], [], [], timeout)
179
202
  end
180
203
 
181
204
  def drain_pipe(pipe)
@@ -1,6 +1,10 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'delayed_job_worker_pool/application'
2
4
  require 'delayed_job_worker_pool/dsl'
5
+ require 'delayed_job_worker_pool/registry'
3
6
  require 'delayed_job_worker_pool/worker'
4
7
  require 'delayed_job_worker_pool/worker_info'
5
8
  require 'delayed_job_worker_pool/worker_pool'
9
+ require 'delayed_job_worker_pool/worker_group_options'
6
10
  require 'delayed_job_worker_pool/version'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: delayed_job_worker_pool
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Joel Turkel
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-06 00:00:00.000000000 Z
11
+ date: 2022-09-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: delayed_job
@@ -31,7 +31,7 @@ dependencies:
31
31
  - !ruby/object:Gem::Version
32
32
  version: '4.2'
33
33
  - !ruby/object:Gem::Dependency
34
- name: delayed_job_active_record
34
+ name: appraisal
35
35
  requirement: !ruby/object:Gem::Requirement
36
36
  requirements:
37
37
  - - ">="
@@ -50,70 +50,132 @@ dependencies:
50
50
  requirements:
51
51
  - - "~>"
52
52
  - !ruby/object:Gem::Version
53
- version: '1.10'
53
+ version: '2.0'
54
54
  type: :development
55
55
  prerelease: false
56
56
  version_requirements: !ruby/object:Gem::Requirement
57
57
  requirements:
58
58
  - - "~>"
59
59
  - !ruby/object:Gem::Version
60
- version: '1.10'
60
+ version: '2.0'
61
+ - !ruby/object:Gem::Dependency
62
+ name: delayed_job_active_record
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - ">="
66
+ - !ruby/object:Gem::Version
67
+ version: '0'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ name: rails
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - ">="
80
+ - !ruby/object:Gem::Version
81
+ version: '6.0'
82
+ - - "<"
83
+ - !ruby/object:Gem::Version
84
+ version: '8'
85
+ type: :development
86
+ prerelease: false
87
+ version_requirements: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '6.0'
92
+ - - "<"
93
+ - !ruby/object:Gem::Version
94
+ version: '8'
61
95
  - !ruby/object:Gem::Dependency
62
96
  name: rake
63
97
  requirement: !ruby/object:Gem::Requirement
64
98
  requirements:
65
99
  - - "~>"
66
100
  - !ruby/object:Gem::Version
67
- version: '10.0'
101
+ version: '13.0'
68
102
  type: :development
69
103
  prerelease: false
70
104
  version_requirements: !ruby/object:Gem::Requirement
71
105
  requirements:
72
106
  - - "~>"
73
107
  - !ruby/object:Gem::Version
74
- version: '10.0'
108
+ version: '13.0'
75
109
  - !ruby/object:Gem::Dependency
76
110
  name: rspec
77
111
  requirement: !ruby/object:Gem::Requirement
78
112
  requirements:
79
113
  - - ">="
80
114
  - !ruby/object:Gem::Version
81
- version: '3.3'
115
+ version: '3.8'
82
116
  type: :development
83
117
  prerelease: false
84
118
  version_requirements: !ruby/object:Gem::Requirement
85
119
  requirements:
86
120
  - - ">="
87
121
  - !ruby/object:Gem::Version
88
- version: '3.3'
122
+ version: '3.8'
89
123
  - !ruby/object:Gem::Dependency
90
- name: sqlite3
124
+ name: rspec_junit_formatter
91
125
  requirement: !ruby/object:Gem::Requirement
92
126
  requirements:
93
127
  - - ">="
94
128
  - !ruby/object:Gem::Version
95
- version: '1.3'
129
+ version: '0'
96
130
  type: :development
97
131
  prerelease: false
98
132
  version_requirements: !ruby/object:Gem::Requirement
99
133
  requirements:
100
134
  - - ">="
101
135
  - !ruby/object:Gem::Version
102
- version: '1.3'
136
+ version: '0'
103
137
  - !ruby/object:Gem::Dependency
104
- name: rails
138
+ name: salsify_rubocop
139
+ requirement: !ruby/object:Gem::Requirement
140
+ requirements:
141
+ - - "~>"
142
+ - !ruby/object:Gem::Version
143
+ version: 1.27.1
144
+ type: :development
145
+ prerelease: false
146
+ version_requirements: !ruby/object:Gem::Requirement
147
+ requirements:
148
+ - - "~>"
149
+ - !ruby/object:Gem::Version
150
+ version: 1.27.1
151
+ - !ruby/object:Gem::Dependency
152
+ name: sprockets
153
+ requirement: !ruby/object:Gem::Requirement
154
+ requirements:
155
+ - - "<"
156
+ - !ruby/object:Gem::Version
157
+ version: '4'
158
+ type: :development
159
+ prerelease: false
160
+ version_requirements: !ruby/object:Gem::Requirement
161
+ requirements:
162
+ - - "<"
163
+ - !ruby/object:Gem::Version
164
+ version: '4'
165
+ - !ruby/object:Gem::Dependency
166
+ name: sqlite3
105
167
  requirement: !ruby/object:Gem::Requirement
106
168
  requirements:
107
169
  - - ">="
108
170
  - !ruby/object:Gem::Version
109
- version: '4.2'
171
+ version: '1.3'
110
172
  type: :development
111
173
  prerelease: false
112
174
  version_requirements: !ruby/object:Gem::Requirement
113
175
  requirements:
114
176
  - - ">="
115
177
  - !ruby/object:Gem::Version
116
- version: '4.2'
178
+ version: '1.3'
117
179
  description:
118
180
  email:
119
181
  - jturkel@salsify.com
@@ -122,8 +184,11 @@ executables:
122
184
  extensions: []
123
185
  extra_rdoc_files: []
124
186
  files:
187
+ - ".circleci/config.yml"
188
+ - ".github/CODEOWNERS"
125
189
  - ".gitignore"
126
- - ".travis.yml"
190
+ - ".rubocop.yml"
191
+ - Appraisals
127
192
  - CHANGELOG.md
128
193
  - Gemfile
129
194
  - LICENSE.txt
@@ -131,17 +196,24 @@ files:
131
196
  - Rakefile
132
197
  - bin/delayed_job_worker_pool
133
198
  - delayed_job_worker_pool.gemspec
199
+ - gemfiles/rails_6.0.gemfile
200
+ - gemfiles/rails_6.1.gemfile
201
+ - gemfiles/rails_7.0.gemfile
134
202
  - lib/delayed_job_worker_pool.rb
135
203
  - lib/delayed_job_worker_pool/application.rb
136
204
  - lib/delayed_job_worker_pool/dsl.rb
205
+ - lib/delayed_job_worker_pool/registry.rb
137
206
  - lib/delayed_job_worker_pool/version.rb
138
207
  - lib/delayed_job_worker_pool/worker.rb
208
+ - lib/delayed_job_worker_pool/worker_group_options.rb
139
209
  - lib/delayed_job_worker_pool/worker_info.rb
140
210
  - lib/delayed_job_worker_pool/worker_pool.rb
141
211
  homepage: https://github.com/salsify/delayed_job_worker_pool
142
212
  licenses:
143
213
  - MIT
144
- metadata: {}
214
+ metadata:
215
+ allowed_push_host: https://rubygems.org
216
+ rubygems_mfa_required: 'true'
145
217
  post_install_message:
146
218
  rdoc_options: []
147
219
  require_paths:
@@ -150,15 +222,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
150
222
  requirements:
151
223
  - - ">="
152
224
  - !ruby/object:Gem::Version
153
- version: '2.0'
225
+ version: '2.7'
154
226
  required_rubygems_version: !ruby/object:Gem::Requirement
155
227
  requirements:
156
228
  - - ">="
157
229
  - !ruby/object:Gem::Version
158
230
  version: '0'
159
231
  requirements: []
160
- rubyforge_project:
161
- rubygems_version: 2.4.7
232
+ rubygems_version: 3.3.7
162
233
  signing_key:
163
234
  specification_version: 4
164
235
  summary: Worker process pooling for Delayed Job
data/.travis.yml DELETED
@@ -1,6 +0,0 @@
1
- language: ruby
2
- rvm:
3
- - 2.2.3
4
- - 2.1.7
5
- - 2.0.0-p647
6
- before_install: gem install bundler -v 1.10.6