ductwork 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4e9cbff3e15bf798c3ff2019f8b5637459b2b495f30c74e3b352bff97afa3b6b
4
- data.tar.gz: 9d7a023b9c0018fe5aa8dc9291981146f0246deb6d8c1b98f658de146b2b78c5
3
+ metadata.gz: 82de3534dfed89267c6a4aaca8e1337841fa81f66c6c42b494d9fddfe6ff4a0e
4
+ data.tar.gz: 334f632bcf121d1bfcddb94046d797ca1e65b2cd74fe19100b1d6d3560b1f05a
5
5
  SHA512:
6
- metadata.gz: 2c019906fc749f7a0b47e3094823f3c867068bcabb0fea945ad47f017ac9ae39f659f5ca2b6a6741a666ad53eb0867720a0ce9ec2b57fac3e188b87ad6055dae
7
- data.tar.gz: 1f584787c2f6588a86e59cbe6a2dac6fdaf014ff5b1b9632dc6816b1648d756267cdff77df7ffe06ba10ba74a060b72e0611a60f863d875a84981612319cc1a0
6
+ metadata.gz: 76ff2b157c3d85f4443178bf9a138c58dcead96c39c10c91cde45af88b7c07447c6cce73558f8f2a961d5fd2b3574fbbd8371442d1bf39bac4fbe30f80f415fa
7
+ data.tar.gz: c8c060ebb944a64a3eb42e086f857f042b944a5bdaf1c07ff2d518409530febd4b68e21634452c861d885dcf6ea4d4cbb27a10b0231d99149a591b7a1c85546c
data/CHANGELOG.md CHANGED
@@ -1,5 +1,23 @@
1
1
  # Ductwork Changelog
2
2
 
3
+ ## [0.20.0]
4
+
5
+ - feat: add "timed_out" enum value to execution result model
6
+ - chore: change log level to warn for thread restart messages
7
+ - chore: add thread name to log messages
8
+ - feat: add optional index argument to pipeline advancer to be used in thread name
9
+ - fix: use correct "role" in log messages
10
+ - feat: support a "thread-only" mode instead of forking processes
11
+ - chore: use more descriptive thread names for job workers
12
+ - chore: refactor to `Ductwork::Processes::JobWorker#join`
13
+ - chore: refactor to `Ductwork::Processes::JobWorker#kill`
14
+ - chore: refactor to `Ductwork::Processes::JobWorker#name`
15
+ - chore: use `RunningContext` instance instead of a boolean in the Supervisor - this running context will likely be shared to all child runners when running in "all thread" mode
16
+ - chore!: officially drop support for ruby 3.1 - BREAKING CHANGE: while this is a breaking change, this shouldn't affect anyone since ruby 3.1 has been EOL for a bit; this is more of a formality
17
+ - chore: update `bundler`
18
+ - chore: change project ruby version to 4.0.1
19
+ - feat: add "forking" configuration with default - this will be used to change the concurreny model on boot, specifically deciding if pipeline advancer and job workers will be forked or created as threads
20
+
3
21
  ## [0.19.0]
4
22
 
5
23
  - chore: bump rails-related dependencies to v8.1.2
@@ -4,6 +4,7 @@ module Ductwork
4
4
  class Configuration # rubocop:todo Metrics/ClassLength
5
5
  DEFAULT_ENV = :default
6
6
  DEFAULT_FILE_PATH = "config/ductwork.yml"
7
+ DEFAULT_FORKING = "default" # fork pipeline advancer and job workers
7
8
  DEFAULT_JOB_WORKER_COUNT = 5 # threads
8
9
  DEFAULT_JOB_WORKER_MAX_RETRY = 3 # attempts
9
10
  DEFAULT_JOB_WORKER_POLLING_TIMEOUT = 1 # second
@@ -48,6 +49,10 @@ module Ductwork
48
49
  r
49
50
  end
50
51
 
52
+ def forking
53
+ config[:forking] || DEFAULT_FORKING
54
+ end
55
+
51
56
  def pipelines
52
57
  raw_pipelines = config[:pipelines] || []
53
58
 
@@ -8,6 +8,7 @@ module Ductwork
8
8
 
9
9
  enum :result_type,
10
10
  success: "success",
11
- failure: "failure"
11
+ failure: "failure",
12
+ timed_out: "timed_out"
12
13
  end
13
14
  end
@@ -26,10 +26,10 @@ module Ductwork
26
26
  expand: "expand",
27
27
  collapse: "collapse"
28
28
 
29
- def self.build_for_execution(pipeline_id, *args, **kwargs)
29
+ def self.build_for_execution(pipeline_id, *, **)
30
30
  instance = allocate
31
31
  instance.instance_variable_set(:@pipeline_id, pipeline_id)
32
- instance.send(:initialize, *args, **kwargs)
32
+ instance.send(:initialize, *, **)
33
33
  instance
34
34
  end
35
35
 
@@ -15,7 +15,7 @@ module Ductwork
15
15
 
16
16
  def start
17
17
  @thread = Thread.new { work_loop }
18
- @thread.name = "ductwork.job_worker.#{id}"
18
+ @thread.name = name
19
19
  end
20
20
 
21
21
  alias restart start
@@ -28,6 +28,19 @@ module Ductwork
28
28
  running_context.shutdown!
29
29
  end
30
30
 
31
+ def kill
32
+ stop
33
+ thread&.kill
34
+ end
35
+
36
+ def join(limit)
37
+ thread&.join(limit)
38
+ end
39
+
40
+ def name
41
+ "ductwork.job_worker.#{pipeline}.#{id}"
42
+ end
43
+
31
44
  private
32
45
 
33
46
  attr_reader :id, :running_context
@@ -67,7 +67,8 @@ module Ductwork
67
67
  Ductwork.logger.debug(
68
68
  msg: "Created new job worker",
69
69
  role: :job_worker_runner,
70
- pipeline: pipeline
70
+ pipeline: pipeline,
71
+ thread: job_worker.name
71
72
  )
72
73
  end
73
74
  end
@@ -87,10 +88,11 @@ module Ductwork
87
88
  if !job_worker.alive?
88
89
  job_worker.restart
89
90
 
90
- Ductwork.logger.info(
91
+ Ductwork.logger.warn(
91
92
  msg: "Restarted job worker",
92
93
  role: :job_worker_runner,
93
- pipeline: job_worker.pipeline
94
+ pipeline: job_worker.pipeline,
95
+ thread: job_worker.name
94
96
  )
95
97
  end
96
98
  end
@@ -132,7 +134,7 @@ module Ductwork
132
134
 
133
135
  # TODO: Maybe make this configurable. If there's a ton of workers
134
136
  # it may not even get to the "later" ones depending on the timeout
135
- job_worker.thread.join(1)
137
+ job_worker.join(1)
136
138
  end
137
139
  end
138
140
  end
@@ -140,11 +142,11 @@ module Ductwork
140
142
  def kill_remaining_job_workers
141
143
  job_workers.each do |job_worker|
142
144
  if job_worker.alive?
143
- job_worker.thread.kill
145
+ job_worker.kill
144
146
  Ductwork.logger.debug(
145
147
  msg: "Killed thread",
146
148
  role: :job_worker_runner,
147
- thread: job_worker.thread.name
149
+ thread: job_worker.name
148
150
  )
149
151
  end
150
152
  end
@@ -9,13 +9,17 @@ module Ductwork
9
9
 
10
10
  def initialize
11
11
  @pipelines = Ductwork.configuration.pipelines
12
- @runner_klass = case Ductwork.configuration.role
13
- when "all"
14
- supervisor_runner
15
- when "advancer"
16
- pipeline_advancer_runner
17
- when "worker"
18
- job_worker_runner
12
+ @runner_klass = if Ductwork.configuration.forking == "default"
13
+ case Ductwork.configuration.role
14
+ when "all"
15
+ process_supervisor_runner
16
+ when "advancer"
17
+ pipeline_advancer_runner
18
+ when "worker"
19
+ job_worker_runner
20
+ end
21
+ else
22
+ thread_supervisor_runner
19
23
  end
20
24
  end
21
25
 
@@ -29,8 +33,12 @@ module Ductwork
29
33
 
30
34
  attr_reader :pipelines, :runner_klass
31
35
 
32
- def supervisor_runner
33
- Ductwork::Processes::SupervisorRunner
36
+ def thread_supervisor_runner
37
+ Ductwork::Processes::ThreadSupervisorRunner
38
+ end
39
+
40
+ def process_supervisor_runner
41
+ Ductwork::Processes::ProcessSupervisorRunner
34
42
  end
35
43
 
36
44
  def pipeline_advancer_runner
@@ -3,14 +3,57 @@
3
3
  module Ductwork
4
4
  module Processes
5
5
  class PipelineAdvancer
6
- def initialize(running_context, klass)
7
- @running_context = running_context
6
+ attr_reader :thread, :last_heartbeat_at, :pipeline
7
+
8
+ def initialize(klass, index = nil)
8
9
  @klass = klass
10
+ @index = index || 0
11
+ @running_context = Ductwork::RunningContext.new
12
+ @last_heartbeat_at = Time.current
13
+ @thread = nil
14
+ end
15
+
16
+ def start
17
+ @thread = Thread.new { work_loop }
18
+ @thread.name = name
19
+ end
20
+
21
+ alias restart start
22
+
23
+ def alive?
24
+ thread&.alive? || false
25
+ end
26
+
27
+ def stop
28
+ running_context.shutdown!
29
+ end
30
+
31
+ def kill
32
+ stop
33
+ thread&.kill
34
+ end
35
+
36
+ def join(limit)
37
+ thread&.join(limit)
9
38
  end
10
39
 
11
- def run # rubocop:todo Metrics/AbcSize, Metrics/MethodLength
40
+ def name
41
+ "ductwork.pipeline_advancer.#{klass}.#{index}"
42
+ end
43
+
44
+ private
45
+
46
+ attr_reader :klass, :index, :running_context
47
+
48
+ def work_loop # rubocop:todo Metrics
12
49
  run_hooks_for(:start)
13
50
 
51
+ Ductwork.logger.debug(
52
+ msg: "Entering main work loop",
53
+ role: :pipeline_advancer,
54
+ pipeline: klass
55
+ )
56
+
14
57
  while running_context.running?
15
58
  id = Ductwork.wrap_with_app_executor do
16
59
  Ductwork::Pipeline
@@ -42,8 +85,8 @@ module Ductwork
42
85
  role: :pipeline_advancer
43
86
  )
44
87
 
45
- pipeline = Ductwork.wrap_with_app_executor do
46
- pipeline = Ductwork::Pipeline.find(id)
88
+ @pipeline = Ductwork.wrap_with_app_executor do
89
+ @pipeline = Ductwork::Pipeline.find(id)
47
90
  pipeline.advance!
48
91
 
49
92
  Ductwork.logger.debug(
@@ -82,16 +125,20 @@ module Ductwork
82
125
  )
83
126
  end
84
127
 
128
+ @last_heartbeat_at = Time.current
129
+
85
130
  sleep(polling_timeout)
86
131
  end
87
132
 
133
+ Ductwork.logger.debug(
134
+ msg: "Shutting down",
135
+ role: :pipeline_advancer,
136
+ pipeline: klass
137
+ )
138
+
88
139
  run_hooks_for(:stop)
89
140
  end
90
141
 
91
- private
92
-
93
- attr_reader :running_context, :klass
94
-
95
142
  def run_hooks_for(event)
96
143
  Ductwork.hooks[:advancer].fetch(event, []).each do |block|
97
144
  Ductwork.wrap_with_app_executor do
@@ -6,7 +6,7 @@ module Ductwork
6
6
  def initialize(*klasses)
7
7
  @klasses = klasses
8
8
  @running_context = Ductwork::RunningContext.new
9
- @threads = create_threads
9
+ @advancers = []
10
10
 
11
11
  Signal.trap(:INT) { running_context.shutdown! }
12
12
  Signal.trap(:TERM) { running_context.shutdown! }
@@ -25,54 +25,65 @@ module Ductwork
25
25
 
26
26
  def run
27
27
  create_process!
28
+ start_pipeline_advancers
29
+
28
30
  Ductwork.logger.debug(
29
31
  msg: "Entering main work loop",
30
- role: :pipeline_advancer_runner
32
+ role: :pipeline_advancer_runner,
33
+ pipelines: klasses
31
34
  )
32
35
 
33
36
  while running_context.running?
34
37
  # TODO: Increase or make configurable
35
38
  sleep(5)
36
- attempt_synchronize_threads
39
+ check_thread_health
37
40
  report_heartbeat!
38
41
  end
39
42
 
40
- shutdown
43
+ shutdown!
41
44
  end
42
45
 
43
46
  private
44
47
 
45
- attr_reader :klasses, :running_context, :threads
48
+ attr_reader :klasses, :running_context, :advancers
46
49
 
47
- def create_threads
48
- klasses.map do |klass|
49
- thread = Thread.new do
50
- Ductwork::Processes::PipelineAdvancer
51
- .new(running_context, klass)
52
- .run
53
- end
54
- thread.name = "ductwork.pipeline_advancer.#{klass}"
50
+ def start_pipeline_advancers
51
+ klasses.each do |klass|
52
+ advancer = Ductwork::Processes::PipelineAdvancer.new(klass)
53
+ advancers.push(advancer)
54
+ advancer.start
55
55
 
56
56
  Ductwork.logger.debug(
57
- msg: "Created new thread",
57
+ msg: "Created new pipeline advancer",
58
58
  role: :pipeline_advancer_runner,
59
- thread: thread.name,
60
- pipeline: klass
59
+ pipeline: klass,
60
+ thread: advancer.name
61
61
  )
62
-
63
- thread
64
62
  end
65
63
  end
66
64
 
67
- def attempt_synchronize_threads
65
+ def check_thread_health
68
66
  Ductwork.logger.debug(
69
- msg: "Attempting to synchronize threads",
70
- role: :pipeline_advancer_runner
67
+ msg: "Checking threads health",
68
+ role: :pipeline_advancer_runner,
69
+ pipelines: klasses
71
70
  )
72
- threads.each { |thread| thread.join(0.1) }
71
+ advancers.each do |advancer|
72
+ if !advancer.alive?
73
+ advancer.restart
74
+
75
+ Ductwork.logger.warn(
76
+ msg: "Restarted pipeline advancer",
77
+ role: :pipeline_advancer_runner,
78
+ pipeline: advancer.pipeline,
79
+ thread: advancer.name
80
+ )
81
+ end
82
+ end
73
83
  Ductwork.logger.debug(
74
- msg: "Synchronizing threads timed out",
75
- role: :pipeline_advancer_runner
84
+ msg: "Checked thread health",
85
+ role: :pipeline_advancer_runner,
86
+ pipelines: klasses
76
87
  )
77
88
  end
78
89
 
@@ -94,9 +105,10 @@ module Ductwork
94
105
  Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
95
106
  end
96
107
 
97
- def shutdown
108
+ def shutdown!
98
109
  log_shutting_down
99
110
  stop_running_context
111
+ advancers.each(&:stop)
100
112
  await_threads_graceful_shutdown
101
113
  kill_remaining_threads
102
114
  delete_process!
@@ -118,25 +130,25 @@ module Ductwork
118
130
  msg: "Attempting graceful shutdown",
119
131
  role: :pipeline_advancer_runner
120
132
  )
121
- while Time.current < deadline && threads.any?(&:alive?)
122
- threads.each do |thread|
133
+ while Time.current < deadline && advancers.any?(&:alive?)
134
+ advancers.each do |advancer|
123
135
  break if Time.current > deadline
124
136
 
125
137
  # TODO: Maybe make this configurable. If there's a ton of workers
126
138
  # it may not even get to the "later" ones depending on the timeout
127
- thread.join(1)
139
+ advancer.join(1)
128
140
  end
129
141
  end
130
142
  end
131
143
 
132
144
  def kill_remaining_threads
133
- threads.each do |thread|
134
- if thread.alive?
135
- thread.kill
145
+ advancers.each do |advancer|
146
+ if advancer.alive?
147
+ advancer.kill
136
148
  Ductwork.logger.debug(
137
149
  msg: "Killed thread",
138
150
  role: :pipeline_advancer_runner,
139
- thread: thread.name
151
+ thread: advancer.name
140
152
  )
141
153
  end
142
154
  end
@@ -2,17 +2,17 @@
2
2
 
3
3
  module Ductwork
4
4
  module Processes
5
- class Supervisor
5
+ class ProcessSupervisor
6
6
  attr_reader :workers
7
7
 
8
8
  def initialize
9
- @running = true
9
+ @running_context = Ductwork::RunningContext.new
10
10
  @workers = []
11
11
 
12
12
  run_hooks_for(:start)
13
13
 
14
- Signal.trap(:INT) { @running = false }
15
- Signal.trap(:TERM) { @running = false }
14
+ Signal.trap(:INT) { @running_context.shutdown! }
15
+ Signal.trap(:TERM) { @running_context.shutdown! }
16
16
  Signal.trap(:TTIN) { puts "No threads to dump" }
17
17
  end
18
18
 
@@ -29,9 +29,13 @@ module Ductwork
29
29
  end
30
30
 
31
31
  def run
32
- Ductwork.logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
32
+ Ductwork.logger.debug(
33
+ msg: "Entering main work loop",
34
+ role: :process_supervisor,
35
+ pid: ::Process.pid
36
+ )
33
37
 
34
- while running
38
+ while running_context.running?
35
39
  sleep(Ductwork.configuration.supervisor_polling_timeout)
36
40
  check_workers
37
41
  end
@@ -40,9 +44,12 @@ module Ductwork
40
44
  end
41
45
 
42
46
  def shutdown
43
- @running = false
47
+ running_context.shutdown!
48
+ Ductwork.logger.debug(
49
+ msg: "Beginning shutdown",
50
+ role: :process_supervisor
51
+ )
44
52
 
45
- Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
46
53
  terminate_gracefully
47
54
  wait_for_workers_to_exit
48
55
  terminate_immediately
@@ -51,10 +58,13 @@ module Ductwork
51
58
 
52
59
  private
53
60
 
54
- attr_reader :running
61
+ attr_reader :running_context
55
62
 
56
63
  def check_workers
57
- Ductwork.logger.debug(msg: "Checking workers are alive", role: :supervisor)
64
+ Ductwork.logger.debug(
65
+ msg: "Checking workers are alive",
66
+ role: :process_supervisor
67
+ )
58
68
 
59
69
  workers.each do |worker|
60
70
  if process_dead?(worker[:pid])
@@ -65,21 +75,24 @@ module Ductwork
65
75
  worker[:pid] = new_pid
66
76
  Ductwork.logger.debug(
67
77
  msg: "Restarted process (#{old_pid}) as (#{new_pid})",
68
- role: :supervisor,
78
+ role: :process_supervisor,
69
79
  old_pid: old_pid,
70
80
  new_pid: new_pid
71
81
  )
72
82
  end
73
83
  end
74
84
 
75
- Ductwork.logger.debug(msg: "All workers are alive or revived", role: :supervisor)
85
+ Ductwork.logger.debug(
86
+ msg: "All workers are alive or revived",
87
+ role: :process_supervisor
88
+ )
76
89
  end
77
90
 
78
91
  def terminate_gracefully
79
92
  workers.each do |worker|
80
93
  Ductwork.logger.debug(
81
94
  msg: "Sending TERM signal to process (#{worker[:pid]})",
82
- role: :supervisor,
95
+ role: :process_supervisor,
83
96
  pid: worker[:pid],
84
97
  signal: :TERM
85
98
  )
@@ -97,7 +110,7 @@ module Ductwork
97
110
  workers[index] = nil
98
111
  Ductwork.logger.debug(
99
112
  msg: "Child process (#{worker[:pid]}) stopped successfully",
100
- role: :supervisor,
113
+ role: :process_supervisor,
101
114
  pid: worker[:pid]
102
115
  )
103
116
  end
@@ -110,7 +123,7 @@ module Ductwork
110
123
  workers.each_with_index do |worker, index|
111
124
  Ductwork.logger.debug(
112
125
  msg: "Sending KILL signal to process (#{worker[:pid]})",
113
- role: :supervisor,
126
+ role: :process_supervisor,
114
127
  pid: worker[:pid],
115
128
  signal: :KILL
116
129
  )
@@ -119,7 +132,7 @@ module Ductwork
119
132
  workers[index] = nil
120
133
  Ductwork.logger.debug(
121
134
  msg: "Child process (#{worker[:pid]}) killed after timeout",
122
- role: :supervisor,
135
+ role: :process_supervisor,
123
136
  pid: worker[:pid]
124
137
  )
125
138
  rescue Errno::ESRCH, Errno::ECHILD
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ductwork
4
+ module Processes
5
+ class ProcessSupervisorRunner
6
+ def initialize(*pipelines)
7
+ @pipelines = pipelines
8
+ @supervisor = Ductwork::Processes::ProcessSupervisor.new
9
+ end
10
+
11
+ def run
12
+ supervisor.add_worker(metadata: { pipelines: }) do
13
+ log_starting_pipline_advancer
14
+ pipline_advancer_runner.new(*pipelines).run
15
+ end
16
+
17
+ pipelines.each do |pipeline|
18
+ supervisor.add_worker(metadata: { pipeline: }) do
19
+ log_starting_job_worker(pipeline)
20
+ job_worker_runner.new(*pipeline).run
21
+ end
22
+ end
23
+
24
+ supervisor.run
25
+ end
26
+
27
+ private
28
+
29
+ attr_reader :pipelines, :supervisor
30
+
31
+ def log_starting_pipline_advancer
32
+ Ductwork.logger.debug(
33
+ msg: "Starting Pipeline Advancer process",
34
+ role: :process_supervisor_runner
35
+ )
36
+ end
37
+
38
+ def pipline_advancer_runner
39
+ Ductwork::Processes::PipelineAdvancerRunner
40
+ end
41
+
42
+ def log_starting_job_worker(pipeline)
43
+ Ductwork.logger.debug(
44
+ msg: "Starting Job Worker Runner process",
45
+ role: :process_supervisor_runner,
46
+ pipeline: pipeline
47
+ )
48
+ end
49
+
50
+ def job_worker_runner
51
+ Ductwork::Processes::JobWorkerRunner
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,142 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ductwork
4
+ module Processes
5
+ class ThreadSupervisor
6
+ attr_reader :workers
7
+
8
+ def initialize
9
+ @running_context = Ductwork::RunningContext.new
10
+ @workers = []
11
+
12
+ run_hooks_for(:start)
13
+
14
+ Signal.trap(:INT) { @running_context.shutdown! }
15
+ Signal.trap(:TERM) { @running_context.shutdown! }
16
+ Signal.trap(:TTIN) do
17
+ Thread.list.each do |thread|
18
+ puts thread.name
19
+ if thread.backtrace
20
+ puts thread.backtrace.join("\n")
21
+ else
22
+ puts "No backtrace to dump"
23
+ end
24
+ puts
25
+ end
26
+ end
27
+ end
28
+
29
+ # TODO: maybe change the whole supervisor interface because this is clunky
30
+ def add_worker(metadata: {}, &block)
31
+ worker = block.call(metadata)
32
+ workers << worker
33
+ worker.start
34
+
35
+ Ductwork.logger.debug(
36
+ msg: "Started supervised thread",
37
+ role: :thread_supervisor,
38
+ thread: worker.name
39
+ )
40
+ end
41
+
42
+ def run
43
+ Ductwork.logger.debug(
44
+ msg: "Entering main work loop",
45
+ role: :thread_supervisor,
46
+ pid: ::Process.pid
47
+ )
48
+
49
+ while running_context.running?
50
+ sleep(Ductwork.configuration.supervisor_polling_timeout)
51
+ check_worker_health
52
+ end
53
+
54
+ shutdown
55
+ end
56
+
57
+ private
58
+
59
+ attr_reader :running_context
60
+
61
+ def check_worker_health
62
+ Ductwork.logger.debug(
63
+ msg: "Checking workers are alive",
64
+ role: :thread_supervisor
65
+ )
66
+
67
+ workers.each do |worker|
68
+ if !worker.alive?
69
+ worker.restart
70
+
71
+ Ductwork.logger.warn(
72
+ msg: "Restarted supervised thread",
73
+ role: :thread_supervisor,
74
+ thread: worker.name
75
+ )
76
+ end
77
+ end
78
+
79
+ Ductwork.logger.debug(
80
+ msg: "Checked workers are alive",
81
+ role: :thread_supervisor
82
+ )
83
+ end
84
+
85
+ def shutdown
86
+ running_context.shutdown!
87
+ log_beginning_shutdown
88
+ workers.each(&:stop)
89
+ await_threads_graceful_shutdown
90
+ kill_remaining_threads
91
+ run_hooks_for(:stop)
92
+ end
93
+
94
+ def log_beginning_shutdown
95
+ Ductwork.logger.debug(
96
+ msg: "Beginning shutdown",
97
+ role: :thread_supervisor
98
+ )
99
+ end
100
+
101
+ def await_threads_graceful_shutdown
102
+ timeout = Ductwork.configuration.supervisor_shutdown_timeout
103
+ deadline = Time.current + timeout
104
+
105
+ Ductwork.logger.debug(
106
+ msg: "Attempting graceful shutdown",
107
+ role: :thread_supervisor
108
+ )
109
+ while Time.current < deadline && workers.any?(&:alive?)
110
+ workers.each do |worker|
111
+ break if Time.current > deadline
112
+
113
+ # TODO: Maybe make this configurable. If there's a ton of workers
114
+ # it may not even get to the "later" ones depending on the timeout
115
+ worker.join(1)
116
+ end
117
+ end
118
+ end
119
+
120
+ def kill_remaining_threads
121
+ workers.each do |worker|
122
+ if worker.alive?
123
+ worker.kill
124
+ Ductwork.logger.debug(
125
+ msg: "Killed supervised thread",
126
+ role: :thread_supervisor,
127
+ thread: worker.name
128
+ )
129
+ end
130
+ end
131
+ end
132
+
133
+ def run_hooks_for(event)
134
+ Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
135
+ Ductwork.wrap_with_app_executor do
136
+ block.call(self)
137
+ end
138
+ end
139
+ end
140
+ end
141
+ end
142
+ end
@@ -0,0 +1,65 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ductwork
4
+ module Processes
5
+ class ThreadSupervisorRunner
6
+ def initialize(*pipelines)
7
+ @pipelines = pipelines
8
+ @supervisor = Ductwork::Processes::ThreadSupervisor.new
9
+ end
10
+
11
+ def run
12
+ if Ductwork.configuration.role.in?(%w[all advancer])
13
+ pipelines.each do |pipeline|
14
+ log_created_pipeline_advancer(pipeline)
15
+ supervisor.add_worker(metadata: { pipeline: }) do
16
+ pipeline_advancer.new(pipeline)
17
+ end
18
+ end
19
+ end
20
+
21
+ if Ductwork.configuration.role.in?(%w[all worker])
22
+ pipelines.each do |pipeline|
23
+ Ductwork.configuration.job_worker_count(pipeline).times do |index|
24
+ log_created_job_worker(pipeline, index)
25
+ supervisor.add_worker(metadata: { pipeline: }) do
26
+ job_worker.new(pipeline, index)
27
+ end
28
+ end
29
+ end
30
+ end
31
+
32
+ supervisor.run
33
+ end
34
+
35
+ private
36
+
37
+ attr_reader :pipelines, :supervisor
38
+
39
+ def log_created_pipeline_advancer(pipeline)
40
+ Ductwork.logger.debug(
41
+ msg: "Created new pipeline advancer",
42
+ role: :thread_supervisor_runner,
43
+ pipeline: pipeline
44
+ )
45
+ end
46
+
47
+ def pipeline_advancer
48
+ Ductwork::Processes::PipelineAdvancer
49
+ end
50
+
51
+ def log_created_job_worker(pipeline, index)
52
+ Ductwork.logger.debug(
53
+ msg: "Created new job worker",
54
+ role: :thread_supervisor_runner,
55
+ pipeline: pipeline,
56
+ index: index
57
+ )
58
+ end
59
+
60
+ def job_worker
61
+ Ductwork::Processes::JobWorker
62
+ end
63
+ end
64
+ end
65
+ end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ductwork
4
- VERSION = "0.19.0"
4
+ VERSION = "0.20.0"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: ductwork
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.19.0
4
+ version: 0.20.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tyler Ewing
@@ -156,8 +156,10 @@ files:
156
156
  - lib/ductwork/processes/launcher.rb
157
157
  - lib/ductwork/processes/pipeline_advancer.rb
158
158
  - lib/ductwork/processes/pipeline_advancer_runner.rb
159
- - lib/ductwork/processes/supervisor.rb
160
- - lib/ductwork/processes/supervisor_runner.rb
159
+ - lib/ductwork/processes/process_supervisor.rb
160
+ - lib/ductwork/processes/process_supervisor_runner.rb
161
+ - lib/ductwork/processes/thread_supervisor.rb
162
+ - lib/ductwork/processes/thread_supervisor_runner.rb
161
163
  - lib/ductwork/running_context.rb
162
164
  - lib/ductwork/testing.rb
163
165
  - lib/ductwork/testing/helpers.rb
@@ -194,7 +196,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
194
196
  requirements:
195
197
  - - ">="
196
198
  - !ruby/object:Gem::Version
197
- version: 3.1.0
199
+ version: 3.2.0
198
200
  required_rubygems_version: !ruby/object:Gem::Requirement
199
201
  requirements:
200
202
  - - ">="
@@ -1,39 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module Ductwork
4
- module Processes
5
- class SupervisorRunner
6
- def initialize(*pipelines)
7
- @pipelines = pipelines
8
- @supervisor = Ductwork::Processes::Supervisor.new
9
- end
10
-
11
- def run
12
- supervisor.add_worker(metadata: { pipelines: }) do
13
- Ductwork.logger.debug(
14
- msg: "Starting Pipeline Advancer process",
15
- role: :supervisor_runner
16
- )
17
- Ductwork::Processes::PipelineAdvancerRunner.new(*pipelines).run
18
- end
19
-
20
- pipelines.each do |pipeline|
21
- supervisor.add_worker(metadata: { pipeline: }) do
22
- Ductwork.logger.debug(
23
- msg: "Starting Job Worker Runner process",
24
- role: :supervisor_runner,
25
- pipeline: pipeline
26
- )
27
- Ductwork::Processes::JobWorkerRunner.new(*pipeline).run
28
- end
29
- end
30
-
31
- supervisor.run
32
- end
33
-
34
- private
35
-
36
- attr_reader :pipelines, :supervisor
37
- end
38
- end
39
- end