ductwork 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/lib/ductwork/configuration.rb +5 -0
- data/lib/ductwork/models/result.rb +2 -1
- data/lib/ductwork/models/step.rb +2 -2
- data/lib/ductwork/processes/job_worker.rb +14 -1
- data/lib/ductwork/processes/job_worker_runner.rb +8 -6
- data/lib/ductwork/processes/launcher.rb +17 -9
- data/lib/ductwork/processes/pipeline_advancer.rb +56 -9
- data/lib/ductwork/processes/pipeline_advancer_runner.rb +44 -32
- data/lib/ductwork/processes/{supervisor.rb → process_supervisor.rb} +29 -16
- data/lib/ductwork/processes/process_supervisor_runner.rb +55 -0
- data/lib/ductwork/processes/thread_supervisor.rb +142 -0
- data/lib/ductwork/processes/thread_supervisor_runner.rb +65 -0
- data/lib/ductwork/version.rb +1 -1
- metadata +6 -4
- data/lib/ductwork/processes/supervisor_runner.rb +0 -39
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 82de3534dfed89267c6a4aaca8e1337841fa81f66c6c42b494d9fddfe6ff4a0e
|
|
4
|
+
data.tar.gz: 334f632bcf121d1bfcddb94046d797ca1e65b2cd74fe19100b1d6d3560b1f05a
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 76ff2b157c3d85f4443178bf9a138c58dcead96c39c10c91cde45af88b7c07447c6cce73558f8f2a961d5fd2b3574fbbd8371442d1bf39bac4fbe30f80f415fa
|
|
7
|
+
data.tar.gz: c8c060ebb944a64a3eb42e086f857f042b944a5bdaf1c07ff2d518409530febd4b68e21634452c861d885dcf6ea4d4cbb27a10b0231d99149a591b7a1c85546c
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Ductwork Changelog
|
|
2
2
|
|
|
3
|
+
## [0.20.0]
|
|
4
|
+
|
|
5
|
+
- feat: add "timed_out" enum value to execution result model
|
|
6
|
+
- chore: change log level to warn for thread restart messages
|
|
7
|
+
- chore: add thread name to log messages
|
|
8
|
+
- feat: add optional index argument to pipeline advancer to be used in thread name
|
|
9
|
+
- fix: use correct "role" in log messages
|
|
10
|
+
- feat: support a "thread-only" mode instead of forking processes
|
|
11
|
+
- chore: use more descriptive thread names for job workers
|
|
12
|
+
- chore: refactor to `Ductwork::Processes::JobWorker#join`
|
|
13
|
+
- chore: refactor to `Ductwork::Processes::JobWorker#kill`
|
|
14
|
+
- chore: refactor to `Ductwork::Processes::JobWorker#name`
|
|
15
|
+
- chore: use `RunningContext` instance instead of a boolean in the Supervisor - this running context will likely be shared to all child runners when running in "all thread" mode
|
|
16
|
+
- chore!: officially drop support for ruby 3.1 - BREAKING CHANGE: while this is a breaking change, this shouldn't affect anyone since ruby 3.1 has been EOL for a bit; this is more of a formality
|
|
17
|
+
- chore: update `bundler`
|
|
18
|
+
- chore: change project ruby version to 4.0.1
|
|
19
|
+
- feat: add "forking" configuration with default - this will be used to change the concurreny model on boot, specifically deciding if pipeline advancer and job workers will be forked or created as threads
|
|
20
|
+
|
|
3
21
|
## [0.19.0]
|
|
4
22
|
|
|
5
23
|
- chore: bump rails-related dependencies to v8.1.2
|
|
@@ -4,6 +4,7 @@ module Ductwork
|
|
|
4
4
|
class Configuration # rubocop:todo Metrics/ClassLength
|
|
5
5
|
DEFAULT_ENV = :default
|
|
6
6
|
DEFAULT_FILE_PATH = "config/ductwork.yml"
|
|
7
|
+
DEFAULT_FORKING = "default" # fork pipeline advancer and job workers
|
|
7
8
|
DEFAULT_JOB_WORKER_COUNT = 5 # threads
|
|
8
9
|
DEFAULT_JOB_WORKER_MAX_RETRY = 3 # attempts
|
|
9
10
|
DEFAULT_JOB_WORKER_POLLING_TIMEOUT = 1 # second
|
|
@@ -48,6 +49,10 @@ module Ductwork
|
|
|
48
49
|
r
|
|
49
50
|
end
|
|
50
51
|
|
|
52
|
+
def forking
|
|
53
|
+
config[:forking] || DEFAULT_FORKING
|
|
54
|
+
end
|
|
55
|
+
|
|
51
56
|
def pipelines
|
|
52
57
|
raw_pipelines = config[:pipelines] || []
|
|
53
58
|
|
data/lib/ductwork/models/step.rb
CHANGED
|
@@ -26,10 +26,10 @@ module Ductwork
|
|
|
26
26
|
expand: "expand",
|
|
27
27
|
collapse: "collapse"
|
|
28
28
|
|
|
29
|
-
def self.build_for_execution(pipeline_id,
|
|
29
|
+
def self.build_for_execution(pipeline_id, *, **)
|
|
30
30
|
instance = allocate
|
|
31
31
|
instance.instance_variable_set(:@pipeline_id, pipeline_id)
|
|
32
|
-
instance.send(:initialize,
|
|
32
|
+
instance.send(:initialize, *, **)
|
|
33
33
|
instance
|
|
34
34
|
end
|
|
35
35
|
|
|
@@ -15,7 +15,7 @@ module Ductwork
|
|
|
15
15
|
|
|
16
16
|
def start
|
|
17
17
|
@thread = Thread.new { work_loop }
|
|
18
|
-
@thread.name =
|
|
18
|
+
@thread.name = name
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
alias restart start
|
|
@@ -28,6 +28,19 @@ module Ductwork
|
|
|
28
28
|
running_context.shutdown!
|
|
29
29
|
end
|
|
30
30
|
|
|
31
|
+
def kill
|
|
32
|
+
stop
|
|
33
|
+
thread&.kill
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def join(limit)
|
|
37
|
+
thread&.join(limit)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def name
|
|
41
|
+
"ductwork.job_worker.#{pipeline}.#{id}"
|
|
42
|
+
end
|
|
43
|
+
|
|
31
44
|
private
|
|
32
45
|
|
|
33
46
|
attr_reader :id, :running_context
|
|
@@ -67,7 +67,8 @@ module Ductwork
|
|
|
67
67
|
Ductwork.logger.debug(
|
|
68
68
|
msg: "Created new job worker",
|
|
69
69
|
role: :job_worker_runner,
|
|
70
|
-
pipeline: pipeline
|
|
70
|
+
pipeline: pipeline,
|
|
71
|
+
thread: job_worker.name
|
|
71
72
|
)
|
|
72
73
|
end
|
|
73
74
|
end
|
|
@@ -87,10 +88,11 @@ module Ductwork
|
|
|
87
88
|
if !job_worker.alive?
|
|
88
89
|
job_worker.restart
|
|
89
90
|
|
|
90
|
-
Ductwork.logger.
|
|
91
|
+
Ductwork.logger.warn(
|
|
91
92
|
msg: "Restarted job worker",
|
|
92
93
|
role: :job_worker_runner,
|
|
93
|
-
pipeline: job_worker.pipeline
|
|
94
|
+
pipeline: job_worker.pipeline,
|
|
95
|
+
thread: job_worker.name
|
|
94
96
|
)
|
|
95
97
|
end
|
|
96
98
|
end
|
|
@@ -132,7 +134,7 @@ module Ductwork
|
|
|
132
134
|
|
|
133
135
|
# TODO: Maybe make this configurable. If there's a ton of workers
|
|
134
136
|
# it may not even get to the "later" ones depending on the timeout
|
|
135
|
-
job_worker.
|
|
137
|
+
job_worker.join(1)
|
|
136
138
|
end
|
|
137
139
|
end
|
|
138
140
|
end
|
|
@@ -140,11 +142,11 @@ module Ductwork
|
|
|
140
142
|
def kill_remaining_job_workers
|
|
141
143
|
job_workers.each do |job_worker|
|
|
142
144
|
if job_worker.alive?
|
|
143
|
-
job_worker.
|
|
145
|
+
job_worker.kill
|
|
144
146
|
Ductwork.logger.debug(
|
|
145
147
|
msg: "Killed thread",
|
|
146
148
|
role: :job_worker_runner,
|
|
147
|
-
thread: job_worker.
|
|
149
|
+
thread: job_worker.name
|
|
148
150
|
)
|
|
149
151
|
end
|
|
150
152
|
end
|
|
@@ -9,13 +9,17 @@ module Ductwork
|
|
|
9
9
|
|
|
10
10
|
def initialize
|
|
11
11
|
@pipelines = Ductwork.configuration.pipelines
|
|
12
|
-
@runner_klass =
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
12
|
+
@runner_klass = if Ductwork.configuration.forking == "default"
|
|
13
|
+
case Ductwork.configuration.role
|
|
14
|
+
when "all"
|
|
15
|
+
process_supervisor_runner
|
|
16
|
+
when "advancer"
|
|
17
|
+
pipeline_advancer_runner
|
|
18
|
+
when "worker"
|
|
19
|
+
job_worker_runner
|
|
20
|
+
end
|
|
21
|
+
else
|
|
22
|
+
thread_supervisor_runner
|
|
19
23
|
end
|
|
20
24
|
end
|
|
21
25
|
|
|
@@ -29,8 +33,12 @@ module Ductwork
|
|
|
29
33
|
|
|
30
34
|
attr_reader :pipelines, :runner_klass
|
|
31
35
|
|
|
32
|
-
def
|
|
33
|
-
Ductwork::Processes::
|
|
36
|
+
def thread_supervisor_runner
|
|
37
|
+
Ductwork::Processes::ThreadSupervisorRunner
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def process_supervisor_runner
|
|
41
|
+
Ductwork::Processes::ProcessSupervisorRunner
|
|
34
42
|
end
|
|
35
43
|
|
|
36
44
|
def pipeline_advancer_runner
|
|
@@ -3,14 +3,57 @@
|
|
|
3
3
|
module Ductwork
|
|
4
4
|
module Processes
|
|
5
5
|
class PipelineAdvancer
|
|
6
|
-
|
|
7
|
-
|
|
6
|
+
attr_reader :thread, :last_heartbeat_at, :pipeline
|
|
7
|
+
|
|
8
|
+
def initialize(klass, index = nil)
|
|
8
9
|
@klass = klass
|
|
10
|
+
@index = index || 0
|
|
11
|
+
@running_context = Ductwork::RunningContext.new
|
|
12
|
+
@last_heartbeat_at = Time.current
|
|
13
|
+
@thread = nil
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def start
|
|
17
|
+
@thread = Thread.new { work_loop }
|
|
18
|
+
@thread.name = name
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
alias restart start
|
|
22
|
+
|
|
23
|
+
def alive?
|
|
24
|
+
thread&.alive? || false
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def stop
|
|
28
|
+
running_context.shutdown!
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def kill
|
|
32
|
+
stop
|
|
33
|
+
thread&.kill
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def join(limit)
|
|
37
|
+
thread&.join(limit)
|
|
9
38
|
end
|
|
10
39
|
|
|
11
|
-
def
|
|
40
|
+
def name
|
|
41
|
+
"ductwork.pipeline_advancer.#{klass}.#{index}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
attr_reader :klass, :index, :running_context
|
|
47
|
+
|
|
48
|
+
def work_loop # rubocop:todo Metrics
|
|
12
49
|
run_hooks_for(:start)
|
|
13
50
|
|
|
51
|
+
Ductwork.logger.debug(
|
|
52
|
+
msg: "Entering main work loop",
|
|
53
|
+
role: :pipeline_advancer,
|
|
54
|
+
pipeline: klass
|
|
55
|
+
)
|
|
56
|
+
|
|
14
57
|
while running_context.running?
|
|
15
58
|
id = Ductwork.wrap_with_app_executor do
|
|
16
59
|
Ductwork::Pipeline
|
|
@@ -42,8 +85,8 @@ module Ductwork
|
|
|
42
85
|
role: :pipeline_advancer
|
|
43
86
|
)
|
|
44
87
|
|
|
45
|
-
pipeline = Ductwork.wrap_with_app_executor do
|
|
46
|
-
pipeline = Ductwork::Pipeline.find(id)
|
|
88
|
+
@pipeline = Ductwork.wrap_with_app_executor do
|
|
89
|
+
@pipeline = Ductwork::Pipeline.find(id)
|
|
47
90
|
pipeline.advance!
|
|
48
91
|
|
|
49
92
|
Ductwork.logger.debug(
|
|
@@ -82,16 +125,20 @@ module Ductwork
|
|
|
82
125
|
)
|
|
83
126
|
end
|
|
84
127
|
|
|
128
|
+
@last_heartbeat_at = Time.current
|
|
129
|
+
|
|
85
130
|
sleep(polling_timeout)
|
|
86
131
|
end
|
|
87
132
|
|
|
133
|
+
Ductwork.logger.debug(
|
|
134
|
+
msg: "Shutting down",
|
|
135
|
+
role: :pipeline_advancer,
|
|
136
|
+
pipeline: klass
|
|
137
|
+
)
|
|
138
|
+
|
|
88
139
|
run_hooks_for(:stop)
|
|
89
140
|
end
|
|
90
141
|
|
|
91
|
-
private
|
|
92
|
-
|
|
93
|
-
attr_reader :running_context, :klass
|
|
94
|
-
|
|
95
142
|
def run_hooks_for(event)
|
|
96
143
|
Ductwork.hooks[:advancer].fetch(event, []).each do |block|
|
|
97
144
|
Ductwork.wrap_with_app_executor do
|
|
@@ -6,7 +6,7 @@ module Ductwork
|
|
|
6
6
|
def initialize(*klasses)
|
|
7
7
|
@klasses = klasses
|
|
8
8
|
@running_context = Ductwork::RunningContext.new
|
|
9
|
-
@
|
|
9
|
+
@advancers = []
|
|
10
10
|
|
|
11
11
|
Signal.trap(:INT) { running_context.shutdown! }
|
|
12
12
|
Signal.trap(:TERM) { running_context.shutdown! }
|
|
@@ -25,54 +25,65 @@ module Ductwork
|
|
|
25
25
|
|
|
26
26
|
def run
|
|
27
27
|
create_process!
|
|
28
|
+
start_pipeline_advancers
|
|
29
|
+
|
|
28
30
|
Ductwork.logger.debug(
|
|
29
31
|
msg: "Entering main work loop",
|
|
30
|
-
role: :pipeline_advancer_runner
|
|
32
|
+
role: :pipeline_advancer_runner,
|
|
33
|
+
pipelines: klasses
|
|
31
34
|
)
|
|
32
35
|
|
|
33
36
|
while running_context.running?
|
|
34
37
|
# TODO: Increase or make configurable
|
|
35
38
|
sleep(5)
|
|
36
|
-
|
|
39
|
+
check_thread_health
|
|
37
40
|
report_heartbeat!
|
|
38
41
|
end
|
|
39
42
|
|
|
40
|
-
shutdown
|
|
43
|
+
shutdown!
|
|
41
44
|
end
|
|
42
45
|
|
|
43
46
|
private
|
|
44
47
|
|
|
45
|
-
attr_reader :klasses, :running_context, :
|
|
48
|
+
attr_reader :klasses, :running_context, :advancers
|
|
46
49
|
|
|
47
|
-
def
|
|
48
|
-
klasses.
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
.run
|
|
53
|
-
end
|
|
54
|
-
thread.name = "ductwork.pipeline_advancer.#{klass}"
|
|
50
|
+
def start_pipeline_advancers
|
|
51
|
+
klasses.each do |klass|
|
|
52
|
+
advancer = Ductwork::Processes::PipelineAdvancer.new(klass)
|
|
53
|
+
advancers.push(advancer)
|
|
54
|
+
advancer.start
|
|
55
55
|
|
|
56
56
|
Ductwork.logger.debug(
|
|
57
|
-
msg: "Created new
|
|
57
|
+
msg: "Created new pipeline advancer",
|
|
58
58
|
role: :pipeline_advancer_runner,
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
pipeline: klass,
|
|
60
|
+
thread: advancer.name
|
|
61
61
|
)
|
|
62
|
-
|
|
63
|
-
thread
|
|
64
62
|
end
|
|
65
63
|
end
|
|
66
64
|
|
|
67
|
-
def
|
|
65
|
+
def check_thread_health
|
|
68
66
|
Ductwork.logger.debug(
|
|
69
|
-
msg: "
|
|
70
|
-
role: :pipeline_advancer_runner
|
|
67
|
+
msg: "Checking threads health",
|
|
68
|
+
role: :pipeline_advancer_runner,
|
|
69
|
+
pipelines: klasses
|
|
71
70
|
)
|
|
72
|
-
|
|
71
|
+
advancers.each do |advancer|
|
|
72
|
+
if !advancer.alive?
|
|
73
|
+
advancer.restart
|
|
74
|
+
|
|
75
|
+
Ductwork.logger.warn(
|
|
76
|
+
msg: "Restarted pipeline advancer",
|
|
77
|
+
role: :pipeline_advancer_runner,
|
|
78
|
+
pipeline: advancer.pipeline,
|
|
79
|
+
thread: advancer.name
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
73
83
|
Ductwork.logger.debug(
|
|
74
|
-
msg: "
|
|
75
|
-
role: :pipeline_advancer_runner
|
|
84
|
+
msg: "Checked thread health",
|
|
85
|
+
role: :pipeline_advancer_runner,
|
|
86
|
+
pipelines: klasses
|
|
76
87
|
)
|
|
77
88
|
end
|
|
78
89
|
|
|
@@ -94,9 +105,10 @@ module Ductwork
|
|
|
94
105
|
Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
|
|
95
106
|
end
|
|
96
107
|
|
|
97
|
-
def shutdown
|
|
108
|
+
def shutdown!
|
|
98
109
|
log_shutting_down
|
|
99
110
|
stop_running_context
|
|
111
|
+
advancers.each(&:stop)
|
|
100
112
|
await_threads_graceful_shutdown
|
|
101
113
|
kill_remaining_threads
|
|
102
114
|
delete_process!
|
|
@@ -118,25 +130,25 @@ module Ductwork
|
|
|
118
130
|
msg: "Attempting graceful shutdown",
|
|
119
131
|
role: :pipeline_advancer_runner
|
|
120
132
|
)
|
|
121
|
-
while Time.current < deadline &&
|
|
122
|
-
|
|
133
|
+
while Time.current < deadline && advancers.any?(&:alive?)
|
|
134
|
+
advancers.each do |advancer|
|
|
123
135
|
break if Time.current > deadline
|
|
124
136
|
|
|
125
137
|
# TODO: Maybe make this configurable. If there's a ton of workers
|
|
126
138
|
# it may not even get to the "later" ones depending on the timeout
|
|
127
|
-
|
|
139
|
+
advancer.join(1)
|
|
128
140
|
end
|
|
129
141
|
end
|
|
130
142
|
end
|
|
131
143
|
|
|
132
144
|
def kill_remaining_threads
|
|
133
|
-
|
|
134
|
-
if
|
|
135
|
-
|
|
145
|
+
advancers.each do |advancer|
|
|
146
|
+
if advancer.alive?
|
|
147
|
+
advancer.kill
|
|
136
148
|
Ductwork.logger.debug(
|
|
137
149
|
msg: "Killed thread",
|
|
138
150
|
role: :pipeline_advancer_runner,
|
|
139
|
-
thread:
|
|
151
|
+
thread: advancer.name
|
|
140
152
|
)
|
|
141
153
|
end
|
|
142
154
|
end
|
|
@@ -2,17 +2,17 @@
|
|
|
2
2
|
|
|
3
3
|
module Ductwork
|
|
4
4
|
module Processes
|
|
5
|
-
class
|
|
5
|
+
class ProcessSupervisor
|
|
6
6
|
attr_reader :workers
|
|
7
7
|
|
|
8
8
|
def initialize
|
|
9
|
-
@
|
|
9
|
+
@running_context = Ductwork::RunningContext.new
|
|
10
10
|
@workers = []
|
|
11
11
|
|
|
12
12
|
run_hooks_for(:start)
|
|
13
13
|
|
|
14
|
-
Signal.trap(:INT) { @
|
|
15
|
-
Signal.trap(:TERM) { @
|
|
14
|
+
Signal.trap(:INT) { @running_context.shutdown! }
|
|
15
|
+
Signal.trap(:TERM) { @running_context.shutdown! }
|
|
16
16
|
Signal.trap(:TTIN) { puts "No threads to dump" }
|
|
17
17
|
end
|
|
18
18
|
|
|
@@ -29,9 +29,13 @@ module Ductwork
|
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def run
|
|
32
|
-
Ductwork.logger.debug(
|
|
32
|
+
Ductwork.logger.debug(
|
|
33
|
+
msg: "Entering main work loop",
|
|
34
|
+
role: :process_supervisor,
|
|
35
|
+
pid: ::Process.pid
|
|
36
|
+
)
|
|
33
37
|
|
|
34
|
-
while running
|
|
38
|
+
while running_context.running?
|
|
35
39
|
sleep(Ductwork.configuration.supervisor_polling_timeout)
|
|
36
40
|
check_workers
|
|
37
41
|
end
|
|
@@ -40,9 +44,12 @@ module Ductwork
|
|
|
40
44
|
end
|
|
41
45
|
|
|
42
46
|
def shutdown
|
|
43
|
-
|
|
47
|
+
running_context.shutdown!
|
|
48
|
+
Ductwork.logger.debug(
|
|
49
|
+
msg: "Beginning shutdown",
|
|
50
|
+
role: :process_supervisor
|
|
51
|
+
)
|
|
44
52
|
|
|
45
|
-
Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
|
|
46
53
|
terminate_gracefully
|
|
47
54
|
wait_for_workers_to_exit
|
|
48
55
|
terminate_immediately
|
|
@@ -51,10 +58,13 @@ module Ductwork
|
|
|
51
58
|
|
|
52
59
|
private
|
|
53
60
|
|
|
54
|
-
attr_reader :
|
|
61
|
+
attr_reader :running_context
|
|
55
62
|
|
|
56
63
|
def check_workers
|
|
57
|
-
Ductwork.logger.debug(
|
|
64
|
+
Ductwork.logger.debug(
|
|
65
|
+
msg: "Checking workers are alive",
|
|
66
|
+
role: :process_supervisor
|
|
67
|
+
)
|
|
58
68
|
|
|
59
69
|
workers.each do |worker|
|
|
60
70
|
if process_dead?(worker[:pid])
|
|
@@ -65,21 +75,24 @@ module Ductwork
|
|
|
65
75
|
worker[:pid] = new_pid
|
|
66
76
|
Ductwork.logger.debug(
|
|
67
77
|
msg: "Restarted process (#{old_pid}) as (#{new_pid})",
|
|
68
|
-
role: :
|
|
78
|
+
role: :process_supervisor,
|
|
69
79
|
old_pid: old_pid,
|
|
70
80
|
new_pid: new_pid
|
|
71
81
|
)
|
|
72
82
|
end
|
|
73
83
|
end
|
|
74
84
|
|
|
75
|
-
Ductwork.logger.debug(
|
|
85
|
+
Ductwork.logger.debug(
|
|
86
|
+
msg: "All workers are alive or revived",
|
|
87
|
+
role: :process_supervisor
|
|
88
|
+
)
|
|
76
89
|
end
|
|
77
90
|
|
|
78
91
|
def terminate_gracefully
|
|
79
92
|
workers.each do |worker|
|
|
80
93
|
Ductwork.logger.debug(
|
|
81
94
|
msg: "Sending TERM signal to process (#{worker[:pid]})",
|
|
82
|
-
role: :
|
|
95
|
+
role: :process_supervisor,
|
|
83
96
|
pid: worker[:pid],
|
|
84
97
|
signal: :TERM
|
|
85
98
|
)
|
|
@@ -97,7 +110,7 @@ module Ductwork
|
|
|
97
110
|
workers[index] = nil
|
|
98
111
|
Ductwork.logger.debug(
|
|
99
112
|
msg: "Child process (#{worker[:pid]}) stopped successfully",
|
|
100
|
-
role: :
|
|
113
|
+
role: :process_supervisor,
|
|
101
114
|
pid: worker[:pid]
|
|
102
115
|
)
|
|
103
116
|
end
|
|
@@ -110,7 +123,7 @@ module Ductwork
|
|
|
110
123
|
workers.each_with_index do |worker, index|
|
|
111
124
|
Ductwork.logger.debug(
|
|
112
125
|
msg: "Sending KILL signal to process (#{worker[:pid]})",
|
|
113
|
-
role: :
|
|
126
|
+
role: :process_supervisor,
|
|
114
127
|
pid: worker[:pid],
|
|
115
128
|
signal: :KILL
|
|
116
129
|
)
|
|
@@ -119,7 +132,7 @@ module Ductwork
|
|
|
119
132
|
workers[index] = nil
|
|
120
133
|
Ductwork.logger.debug(
|
|
121
134
|
msg: "Child process (#{worker[:pid]}) killed after timeout",
|
|
122
|
-
role: :
|
|
135
|
+
role: :process_supervisor,
|
|
123
136
|
pid: worker[:pid]
|
|
124
137
|
)
|
|
125
138
|
rescue Errno::ESRCH, Errno::ECHILD
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ductwork
|
|
4
|
+
module Processes
|
|
5
|
+
class ProcessSupervisorRunner
|
|
6
|
+
def initialize(*pipelines)
|
|
7
|
+
@pipelines = pipelines
|
|
8
|
+
@supervisor = Ductwork::Processes::ProcessSupervisor.new
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run
|
|
12
|
+
supervisor.add_worker(metadata: { pipelines: }) do
|
|
13
|
+
log_starting_pipline_advancer
|
|
14
|
+
pipline_advancer_runner.new(*pipelines).run
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
pipelines.each do |pipeline|
|
|
18
|
+
supervisor.add_worker(metadata: { pipeline: }) do
|
|
19
|
+
log_starting_job_worker(pipeline)
|
|
20
|
+
job_worker_runner.new(*pipeline).run
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
supervisor.run
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
attr_reader :pipelines, :supervisor
|
|
30
|
+
|
|
31
|
+
def log_starting_pipline_advancer
|
|
32
|
+
Ductwork.logger.debug(
|
|
33
|
+
msg: "Starting Pipeline Advancer process",
|
|
34
|
+
role: :process_supervisor_runner
|
|
35
|
+
)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def pipline_advancer_runner
|
|
39
|
+
Ductwork::Processes::PipelineAdvancerRunner
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def log_starting_job_worker(pipeline)
|
|
43
|
+
Ductwork.logger.debug(
|
|
44
|
+
msg: "Starting Job Worker Runner process",
|
|
45
|
+
role: :process_supervisor_runner,
|
|
46
|
+
pipeline: pipeline
|
|
47
|
+
)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def job_worker_runner
|
|
51
|
+
Ductwork::Processes::JobWorkerRunner
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ductwork
|
|
4
|
+
module Processes
|
|
5
|
+
class ThreadSupervisor
|
|
6
|
+
attr_reader :workers
|
|
7
|
+
|
|
8
|
+
def initialize
|
|
9
|
+
@running_context = Ductwork::RunningContext.new
|
|
10
|
+
@workers = []
|
|
11
|
+
|
|
12
|
+
run_hooks_for(:start)
|
|
13
|
+
|
|
14
|
+
Signal.trap(:INT) { @running_context.shutdown! }
|
|
15
|
+
Signal.trap(:TERM) { @running_context.shutdown! }
|
|
16
|
+
Signal.trap(:TTIN) do
|
|
17
|
+
Thread.list.each do |thread|
|
|
18
|
+
puts thread.name
|
|
19
|
+
if thread.backtrace
|
|
20
|
+
puts thread.backtrace.join("\n")
|
|
21
|
+
else
|
|
22
|
+
puts "No backtrace to dump"
|
|
23
|
+
end
|
|
24
|
+
puts
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# TODO: maybe change the whole supervisor interface because this is clunky
|
|
30
|
+
def add_worker(metadata: {}, &block)
|
|
31
|
+
worker = block.call(metadata)
|
|
32
|
+
workers << worker
|
|
33
|
+
worker.start
|
|
34
|
+
|
|
35
|
+
Ductwork.logger.debug(
|
|
36
|
+
msg: "Started supervised thread",
|
|
37
|
+
role: :thread_supervisor,
|
|
38
|
+
thread: worker.name
|
|
39
|
+
)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def run
|
|
43
|
+
Ductwork.logger.debug(
|
|
44
|
+
msg: "Entering main work loop",
|
|
45
|
+
role: :thread_supervisor,
|
|
46
|
+
pid: ::Process.pid
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
while running_context.running?
|
|
50
|
+
sleep(Ductwork.configuration.supervisor_polling_timeout)
|
|
51
|
+
check_worker_health
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
shutdown
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
private
|
|
58
|
+
|
|
59
|
+
attr_reader :running_context
|
|
60
|
+
|
|
61
|
+
def check_worker_health
|
|
62
|
+
Ductwork.logger.debug(
|
|
63
|
+
msg: "Checking workers are alive",
|
|
64
|
+
role: :thread_supervisor
|
|
65
|
+
)
|
|
66
|
+
|
|
67
|
+
workers.each do |worker|
|
|
68
|
+
if !worker.alive?
|
|
69
|
+
worker.restart
|
|
70
|
+
|
|
71
|
+
Ductwork.logger.warn(
|
|
72
|
+
msg: "Restarted supervised thread",
|
|
73
|
+
role: :thread_supervisor,
|
|
74
|
+
thread: worker.name
|
|
75
|
+
)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
Ductwork.logger.debug(
|
|
80
|
+
msg: "Checked workers are alive",
|
|
81
|
+
role: :thread_supervisor
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def shutdown
|
|
86
|
+
running_context.shutdown!
|
|
87
|
+
log_beginning_shutdown
|
|
88
|
+
workers.each(&:stop)
|
|
89
|
+
await_threads_graceful_shutdown
|
|
90
|
+
kill_remaining_threads
|
|
91
|
+
run_hooks_for(:stop)
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def log_beginning_shutdown
|
|
95
|
+
Ductwork.logger.debug(
|
|
96
|
+
msg: "Beginning shutdown",
|
|
97
|
+
role: :thread_supervisor
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def await_threads_graceful_shutdown
|
|
102
|
+
timeout = Ductwork.configuration.supervisor_shutdown_timeout
|
|
103
|
+
deadline = Time.current + timeout
|
|
104
|
+
|
|
105
|
+
Ductwork.logger.debug(
|
|
106
|
+
msg: "Attempting graceful shutdown",
|
|
107
|
+
role: :thread_supervisor
|
|
108
|
+
)
|
|
109
|
+
while Time.current < deadline && workers.any?(&:alive?)
|
|
110
|
+
workers.each do |worker|
|
|
111
|
+
break if Time.current > deadline
|
|
112
|
+
|
|
113
|
+
# TODO: Maybe make this configurable. If there's a ton of workers
|
|
114
|
+
# it may not even get to the "later" ones depending on the timeout
|
|
115
|
+
worker.join(1)
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def kill_remaining_threads
|
|
121
|
+
workers.each do |worker|
|
|
122
|
+
if worker.alive?
|
|
123
|
+
worker.kill
|
|
124
|
+
Ductwork.logger.debug(
|
|
125
|
+
msg: "Killed supervised thread",
|
|
126
|
+
role: :thread_supervisor,
|
|
127
|
+
thread: worker.name
|
|
128
|
+
)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def run_hooks_for(event)
|
|
134
|
+
Ductwork.hooks[:supervisor].fetch(event, []).each do |block|
|
|
135
|
+
Ductwork.wrap_with_app_executor do
|
|
136
|
+
block.call(self)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ductwork
|
|
4
|
+
module Processes
|
|
5
|
+
class ThreadSupervisorRunner
|
|
6
|
+
def initialize(*pipelines)
|
|
7
|
+
@pipelines = pipelines
|
|
8
|
+
@supervisor = Ductwork::Processes::ThreadSupervisor.new
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def run
|
|
12
|
+
if Ductwork.configuration.role.in?(%w[all advancer])
|
|
13
|
+
pipelines.each do |pipeline|
|
|
14
|
+
log_created_pipeline_advancer(pipeline)
|
|
15
|
+
supervisor.add_worker(metadata: { pipeline: }) do
|
|
16
|
+
pipeline_advancer.new(pipeline)
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
if Ductwork.configuration.role.in?(%w[all worker])
|
|
22
|
+
pipelines.each do |pipeline|
|
|
23
|
+
Ductwork.configuration.job_worker_count(pipeline).times do |index|
|
|
24
|
+
log_created_job_worker(pipeline, index)
|
|
25
|
+
supervisor.add_worker(metadata: { pipeline: }) do
|
|
26
|
+
job_worker.new(pipeline, index)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
supervisor.run
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
attr_reader :pipelines, :supervisor
|
|
38
|
+
|
|
39
|
+
def log_created_pipeline_advancer(pipeline)
|
|
40
|
+
Ductwork.logger.debug(
|
|
41
|
+
msg: "Created new pipeline advancer",
|
|
42
|
+
role: :thread_supervisor_runner,
|
|
43
|
+
pipeline: pipeline
|
|
44
|
+
)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def pipeline_advancer
|
|
48
|
+
Ductwork::Processes::PipelineAdvancer
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def log_created_job_worker(pipeline, index)
|
|
52
|
+
Ductwork.logger.debug(
|
|
53
|
+
msg: "Created new job worker",
|
|
54
|
+
role: :thread_supervisor_runner,
|
|
55
|
+
pipeline: pipeline,
|
|
56
|
+
index: index
|
|
57
|
+
)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def job_worker
|
|
61
|
+
Ductwork::Processes::JobWorker
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
data/lib/ductwork/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ductwork
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.20.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Tyler Ewing
|
|
@@ -156,8 +156,10 @@ files:
|
|
|
156
156
|
- lib/ductwork/processes/launcher.rb
|
|
157
157
|
- lib/ductwork/processes/pipeline_advancer.rb
|
|
158
158
|
- lib/ductwork/processes/pipeline_advancer_runner.rb
|
|
159
|
-
- lib/ductwork/processes/
|
|
160
|
-
- lib/ductwork/processes/
|
|
159
|
+
- lib/ductwork/processes/process_supervisor.rb
|
|
160
|
+
- lib/ductwork/processes/process_supervisor_runner.rb
|
|
161
|
+
- lib/ductwork/processes/thread_supervisor.rb
|
|
162
|
+
- lib/ductwork/processes/thread_supervisor_runner.rb
|
|
161
163
|
- lib/ductwork/running_context.rb
|
|
162
164
|
- lib/ductwork/testing.rb
|
|
163
165
|
- lib/ductwork/testing/helpers.rb
|
|
@@ -194,7 +196,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
194
196
|
requirements:
|
|
195
197
|
- - ">="
|
|
196
198
|
- !ruby/object:Gem::Version
|
|
197
|
-
version: 3.
|
|
199
|
+
version: 3.2.0
|
|
198
200
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
199
201
|
requirements:
|
|
200
202
|
- - ">="
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
module Ductwork
|
|
4
|
-
module Processes
|
|
5
|
-
class SupervisorRunner
|
|
6
|
-
def initialize(*pipelines)
|
|
7
|
-
@pipelines = pipelines
|
|
8
|
-
@supervisor = Ductwork::Processes::Supervisor.new
|
|
9
|
-
end
|
|
10
|
-
|
|
11
|
-
def run
|
|
12
|
-
supervisor.add_worker(metadata: { pipelines: }) do
|
|
13
|
-
Ductwork.logger.debug(
|
|
14
|
-
msg: "Starting Pipeline Advancer process",
|
|
15
|
-
role: :supervisor_runner
|
|
16
|
-
)
|
|
17
|
-
Ductwork::Processes::PipelineAdvancerRunner.new(*pipelines).run
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
pipelines.each do |pipeline|
|
|
21
|
-
supervisor.add_worker(metadata: { pipeline: }) do
|
|
22
|
-
Ductwork.logger.debug(
|
|
23
|
-
msg: "Starting Job Worker Runner process",
|
|
24
|
-
role: :supervisor_runner,
|
|
25
|
-
pipeline: pipeline
|
|
26
|
-
)
|
|
27
|
-
Ductwork::Processes::JobWorkerRunner.new(*pipeline).run
|
|
28
|
-
end
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
supervisor.run
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
private
|
|
35
|
-
|
|
36
|
-
attr_reader :pipelines, :supervisor
|
|
37
|
-
end
|
|
38
|
-
end
|
|
39
|
-
end
|