ductwork 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -0
- data/README.md +1 -1
- data/app/models/ductwork/job.rb +6 -10
- data/app/models/ductwork/pipeline.rb +10 -10
- data/lib/ductwork/cli.rb +6 -6
- data/lib/ductwork/configuration.rb +0 -1
- data/lib/ductwork/dsl/definition_builder.rb +16 -13
- data/lib/ductwork/engine.rb +1 -1
- data/lib/ductwork/processes/job_worker.rb +4 -8
- data/lib/ductwork/processes/job_worker_runner.rb +9 -13
- data/lib/ductwork/processes/pipeline_advancer.rb +4 -8
- data/lib/ductwork/processes/pipeline_advancer_runner.rb +10 -14
- data/lib/ductwork/processes/supervisor.rb +10 -14
- data/lib/ductwork/processes/supervisor_runner.rb +2 -3
- data/lib/ductwork/version.rb +1 -1
- data/lib/ductwork.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ab38ab3c5ce61b7fd6d48eac93d93bcaa33b3476a7b8b6e126403da7b4c2ea04
|
|
4
|
+
data.tar.gz: 7f3c09913cf33f5f2d6b42f2f3b786b09b35d343d1277262f77b0f9ff666b653
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cd1bbacdd428ed77064e526a7a2bfe639aad474c19d002417b5d3893856cef916baaba805124e1f4042c90ce972328f3de8c9747c88a96d8b3a60d88a26b183a
|
|
7
|
+
data.tar.gz: cef8abdee38bea2530ba058fde0062699fc53f190e3e67deb8c369f7c1badae7a7e9538f11ce6fc8f1bce488e15a9ab97271bf6fbc0c59f38f8eea1187f376e8
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
1
|
# Ductwork Changelog
|
|
2
2
|
|
|
3
|
+
## [0.3.1]
|
|
4
|
+
|
|
5
|
+
- chore: bump dependencies and update necessary files
|
|
6
|
+
- chore: update email address in gemspec
|
|
7
|
+
- chore: move `logger` out of config to top-level `Ductwork` module
|
|
8
|
+
- chore: promote `Ductwork::Pipeline#parsed_definition` to a public method
|
|
9
|
+
- fix: raise when trying to collapse a most recently divided pipeline and vice versa
|
|
10
|
+
|
|
3
11
|
## [0.3.0]
|
|
4
12
|
|
|
5
13
|
- fix: correctly create collapsing and combining steps and jobs for complex pipelines
|
data/README.md
CHANGED
|
@@ -103,7 +103,7 @@ end
|
|
|
103
103
|
|
|
104
104
|
**Important:** Return values must be JSON-serializable.
|
|
105
105
|
|
|
106
|
-
See [Defining Pipelines](https://docs.getductwork.io/
|
|
106
|
+
See [Defining Pipelines](https://docs.getductwork.io/getting-started/defining-pipelines.html) for detailed documentation.
|
|
107
107
|
|
|
108
108
|
### 4. Run Ductwork
|
|
109
109
|
|
data/app/models/ductwork/job.rb
CHANGED
|
@@ -39,7 +39,7 @@ module Ductwork
|
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
if rows_updated == 1
|
|
42
|
-
Ductwork.
|
|
42
|
+
Ductwork.logger.debug(
|
|
43
43
|
msg: "Job claimed",
|
|
44
44
|
role: :job_worker,
|
|
45
45
|
process_id: process_id,
|
|
@@ -49,7 +49,7 @@ module Ductwork
|
|
|
49
49
|
.joins(executions: :availability)
|
|
50
50
|
.find_by(ductwork_availabilities: { id:, process_id: })
|
|
51
51
|
else
|
|
52
|
-
Ductwork.
|
|
52
|
+
Ductwork.logger.debug(
|
|
53
53
|
msg: "Did not claim job, avoided race condition",
|
|
54
54
|
role: :job_worker,
|
|
55
55
|
process_id: process_id,
|
|
@@ -78,7 +78,7 @@ module Ductwork
|
|
|
78
78
|
j
|
|
79
79
|
end
|
|
80
80
|
|
|
81
|
-
Ductwork.
|
|
81
|
+
Ductwork.logger.info(
|
|
82
82
|
msg: "Job enqueued",
|
|
83
83
|
job_id: job.id,
|
|
84
84
|
job_klass: job.klass
|
|
@@ -90,7 +90,7 @@ module Ductwork
|
|
|
90
90
|
def execute(pipeline)
|
|
91
91
|
# i don't _really_ like this, but it should be fine for now...
|
|
92
92
|
execution = executions.order(:created_at).last
|
|
93
|
-
logger.debug(
|
|
93
|
+
Ductwork.logger.debug(
|
|
94
94
|
msg: "Executing job",
|
|
95
95
|
role: :job_worker,
|
|
96
96
|
pipeline: pipeline,
|
|
@@ -111,7 +111,7 @@ module Ductwork
|
|
|
111
111
|
execution_failed!(execution, run, e)
|
|
112
112
|
result = "failure"
|
|
113
113
|
ensure
|
|
114
|
-
logger.info(
|
|
114
|
+
Ductwork.logger.info(
|
|
115
115
|
msg: "Job executed",
|
|
116
116
|
pipeline: pipeline,
|
|
117
117
|
job_id: id,
|
|
@@ -130,10 +130,6 @@ module Ductwork
|
|
|
130
130
|
|
|
131
131
|
private
|
|
132
132
|
|
|
133
|
-
def logger
|
|
134
|
-
Ductwork.configuration.logger
|
|
135
|
-
end
|
|
136
|
-
|
|
137
133
|
def execution_succeeded!(execution, run, output_payload)
|
|
138
134
|
payload = JSON.dump({ payload: output_payload })
|
|
139
135
|
|
|
@@ -176,7 +172,7 @@ module Ductwork
|
|
|
176
172
|
end
|
|
177
173
|
end
|
|
178
174
|
|
|
179
|
-
logger.warn(
|
|
175
|
+
Ductwork.logger.warn(
|
|
180
176
|
msg: "Job errored",
|
|
181
177
|
error_klass: error.class.name,
|
|
182
178
|
error_message: error.message,
|
|
@@ -76,7 +76,7 @@ module Ductwork
|
|
|
76
76
|
p
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
Ductwork.
|
|
79
|
+
Ductwork.logger.info(
|
|
80
80
|
msg: "Pipeline triggered",
|
|
81
81
|
pipeline_id: pipeline.id,
|
|
82
82
|
role: :application
|
|
@@ -107,6 +107,10 @@ module Ductwork
|
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
+
def parsed_definition
|
|
111
|
+
@parsed_definition ||= JSON.parse(definition).with_indifferent_access
|
|
112
|
+
end
|
|
113
|
+
|
|
110
114
|
private
|
|
111
115
|
|
|
112
116
|
def create_step_and_enqueue_job(klass:, step_type:, input_arg:)
|
|
@@ -116,17 +120,13 @@ module Ductwork
|
|
|
116
120
|
Ductwork::Job.enqueue(next_step, input_arg)
|
|
117
121
|
end
|
|
118
122
|
|
|
119
|
-
def parsed_definition
|
|
120
|
-
@parsed_definition ||= JSON.parse(definition).with_indifferent_access
|
|
121
|
-
end
|
|
122
|
-
|
|
123
123
|
def conditionally_complete_pipeline(advancing)
|
|
124
124
|
advancing.update!(status: :completed, completed_at: Time.current)
|
|
125
125
|
|
|
126
126
|
if steps.where(status: %w[in_progress pending]).none?
|
|
127
127
|
update!(status: :completed, completed_at: Time.current)
|
|
128
128
|
|
|
129
|
-
Ductwork.
|
|
129
|
+
Ductwork.logger.info(
|
|
130
130
|
msg: "Pipeline completed",
|
|
131
131
|
pipeline_id: id,
|
|
132
132
|
role: :pipeline_advancer
|
|
@@ -167,7 +167,7 @@ module Ductwork
|
|
|
167
167
|
elsif step_type == "expand"
|
|
168
168
|
expand_to_next_steps(step_type, advancing, edge)
|
|
169
169
|
else
|
|
170
|
-
Ductwork.
|
|
170
|
+
Ductwork.logger.error(
|
|
171
171
|
msg: "Invalid step type",
|
|
172
172
|
step_type: step_type,
|
|
173
173
|
pipeline_id: id,
|
|
@@ -193,7 +193,7 @@ module Ductwork
|
|
|
193
193
|
if steps.where(status: %w[pending in_progress], klass: edges.keys).none?
|
|
194
194
|
combine_next_steps(edges, advancing)
|
|
195
195
|
else
|
|
196
|
-
Ductwork.
|
|
196
|
+
Ductwork.logger.debug(
|
|
197
197
|
msg: "Not all divided steps have completed; not combining",
|
|
198
198
|
pipeline_id: id,
|
|
199
199
|
role: :pipeline_advancer
|
|
@@ -232,7 +232,7 @@ module Ductwork
|
|
|
232
232
|
if steps.where(status: %w[pending in_progress], klass: step_klass).none?
|
|
233
233
|
collapse_next_steps(edge[:to].sole, advancing)
|
|
234
234
|
else
|
|
235
|
-
Ductwork.
|
|
235
|
+
Ductwork.logger.debug(
|
|
236
236
|
msg: "Not all expanded steps have completed; not collapsing",
|
|
237
237
|
pipeline_id: id,
|
|
238
238
|
role: :pipeline_advancer
|
|
@@ -255,7 +255,7 @@ module Ductwork
|
|
|
255
255
|
end
|
|
256
256
|
|
|
257
257
|
def log_pipeline_advanced(edges)
|
|
258
|
-
Ductwork.
|
|
258
|
+
Ductwork.logger.info(
|
|
259
259
|
msg: "Pipeline advanced",
|
|
260
260
|
pipeline_id: id,
|
|
261
261
|
transitions: edges.map { |_, v| v.dig(-1, :type) },
|
data/lib/ductwork/cli.rb
CHANGED
|
@@ -8,12 +8,12 @@ module Ductwork
|
|
|
8
8
|
def start!(args)
|
|
9
9
|
options = parse_options(args)
|
|
10
10
|
Ductwork.configuration = Configuration.new(**options)
|
|
11
|
-
Ductwork.
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
Ductwork.
|
|
11
|
+
Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
|
|
12
|
+
Rails.logger
|
|
13
|
+
else
|
|
14
|
+
Ductwork::Configuration::DEFAULT_LOGGER
|
|
15
|
+
end
|
|
16
|
+
Ductwork.logger.level = Ductwork.configuration.logger_level
|
|
17
17
|
|
|
18
18
|
Ductwork::Processes::SupervisorRunner.start!
|
|
19
19
|
end
|
|
@@ -12,8 +12,7 @@ module Ductwork
|
|
|
12
12
|
nodes: [],
|
|
13
13
|
edges: {},
|
|
14
14
|
}
|
|
15
|
-
@
|
|
16
|
-
@expansions = 0
|
|
15
|
+
@divergences = []
|
|
17
16
|
@last_nodes = []
|
|
18
17
|
end
|
|
19
18
|
|
|
@@ -42,7 +41,7 @@ module Ductwork
|
|
|
42
41
|
add_edge_to_last_nodes(*to, type: :divide)
|
|
43
42
|
add_new_nodes(*to)
|
|
44
43
|
|
|
45
|
-
|
|
44
|
+
divergences.push(:divide)
|
|
46
45
|
|
|
47
46
|
if block_given?
|
|
48
47
|
branches = to.map do |klass|
|
|
@@ -59,9 +58,9 @@ module Ductwork
|
|
|
59
58
|
def combine(into:)
|
|
60
59
|
validate_classes!(into)
|
|
61
60
|
validate_definition_started!(action: "combining steps")
|
|
62
|
-
|
|
61
|
+
validate_can_combine!
|
|
63
62
|
|
|
64
|
-
|
|
63
|
+
divergences.pop
|
|
65
64
|
|
|
66
65
|
last_nodes = definition[:nodes].reverse.select do |node|
|
|
67
66
|
definition[:edges][node].empty?
|
|
@@ -83,7 +82,7 @@ module Ductwork
|
|
|
83
82
|
add_edge_to_last_nodes(to, type: :expand)
|
|
84
83
|
add_new_nodes(to)
|
|
85
84
|
|
|
86
|
-
|
|
85
|
+
divergences.push(:expand)
|
|
87
86
|
|
|
88
87
|
self
|
|
89
88
|
end
|
|
@@ -91,11 +90,11 @@ module Ductwork
|
|
|
91
90
|
def collapse(into:)
|
|
92
91
|
validate_classes!(into)
|
|
93
92
|
validate_definition_started!(action: "collapsing steps")
|
|
94
|
-
|
|
93
|
+
validate_can_collapse!
|
|
95
94
|
add_edge_to_last_nodes(into, type: :collapse)
|
|
96
95
|
add_new_nodes(into)
|
|
97
96
|
|
|
98
|
-
|
|
97
|
+
divergences.pop
|
|
99
98
|
|
|
100
99
|
self
|
|
101
100
|
end
|
|
@@ -118,7 +117,7 @@ module Ductwork
|
|
|
118
117
|
|
|
119
118
|
private
|
|
120
119
|
|
|
121
|
-
attr_reader :definition, :
|
|
120
|
+
attr_reader :definition, :last_nodes, :divergences
|
|
122
121
|
|
|
123
122
|
def validate_classes!(klasses)
|
|
124
123
|
valid = Array(klasses).all? do |klass|
|
|
@@ -150,15 +149,19 @@ module Ductwork
|
|
|
150
149
|
end
|
|
151
150
|
end
|
|
152
151
|
|
|
153
|
-
def
|
|
154
|
-
if
|
|
152
|
+
def validate_can_combine!
|
|
153
|
+
if divergences.empty?
|
|
155
154
|
raise CombineError, "Must divide pipeline definition before combining steps"
|
|
155
|
+
elsif divergences[-1] != :divide
|
|
156
|
+
raise CombineError, "Ambiguous combine on most recently expanded definition"
|
|
156
157
|
end
|
|
157
158
|
end
|
|
158
159
|
|
|
159
|
-
def
|
|
160
|
-
if
|
|
160
|
+
def validate_can_collapse!
|
|
161
|
+
if divergences.empty?
|
|
161
162
|
raise CollapseError, "Must expand pipeline definition before collapsing steps"
|
|
163
|
+
elsif divergences[-1] != :expand
|
|
164
|
+
raise CollapseError, "Ambiguous collapse on most recently divided definition"
|
|
162
165
|
end
|
|
163
166
|
end
|
|
164
167
|
|
data/lib/ductwork/engine.rb
CHANGED
|
@@ -8,7 +8,7 @@ module Ductwork
|
|
|
8
8
|
|
|
9
9
|
initializer "ductwork.configure" do
|
|
10
10
|
Ductwork.configuration ||= Ductwork::Configuration.new
|
|
11
|
-
Ductwork.
|
|
11
|
+
Ductwork.logger ||= Ductwork::Configuration::DEFAULT_LOGGER
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
initializer "ductwork.validate_definitions", after: :load_config_initializers do
|
|
@@ -10,13 +10,13 @@ module Ductwork
|
|
|
10
10
|
|
|
11
11
|
def run
|
|
12
12
|
run_hooks_for(:start)
|
|
13
|
-
logger.debug(
|
|
13
|
+
Ductwork.logger.debug(
|
|
14
14
|
msg: "Entering main work loop",
|
|
15
15
|
role: :job_worker,
|
|
16
16
|
pipeline: pipeline
|
|
17
17
|
)
|
|
18
18
|
while running_context.running?
|
|
19
|
-
logger.debug(
|
|
19
|
+
Ductwork.logger.debug(
|
|
20
20
|
msg: "Attempting to claim job",
|
|
21
21
|
role: :job_worker,
|
|
22
22
|
pipeline: pipeline
|
|
@@ -30,7 +30,7 @@ module Ductwork
|
|
|
30
30
|
job.execute(pipeline)
|
|
31
31
|
end
|
|
32
32
|
else
|
|
33
|
-
logger.debug(
|
|
33
|
+
Ductwork.logger.debug(
|
|
34
34
|
msg: "No job to claim, looping",
|
|
35
35
|
role: :job_worker,
|
|
36
36
|
pipeline: pipeline
|
|
@@ -47,7 +47,7 @@ module Ductwork
|
|
|
47
47
|
attr_reader :pipeline, :running_context
|
|
48
48
|
|
|
49
49
|
def shutdown
|
|
50
|
-
logger.debug(
|
|
50
|
+
Ductwork.logger.debug(
|
|
51
51
|
msg: "Shutting down",
|
|
52
52
|
role: :job_worker,
|
|
53
53
|
pipeline: pipeline
|
|
@@ -62,10 +62,6 @@ module Ductwork
|
|
|
62
62
|
end
|
|
63
63
|
end
|
|
64
64
|
end
|
|
65
|
-
|
|
66
|
-
def logger
|
|
67
|
-
Ductwork.configuration.logger
|
|
68
|
-
end
|
|
69
65
|
end
|
|
70
66
|
end
|
|
71
67
|
end
|
|
@@ -25,7 +25,7 @@ module Ductwork
|
|
|
25
25
|
|
|
26
26
|
def run
|
|
27
27
|
create_process!
|
|
28
|
-
logger.debug(
|
|
28
|
+
Ductwork.logger.debug(
|
|
29
29
|
msg: "Entering main work loop",
|
|
30
30
|
role: :job_worker_runner,
|
|
31
31
|
pipeline: pipeline
|
|
@@ -55,7 +55,7 @@ module Ductwork
|
|
|
55
55
|
pipeline,
|
|
56
56
|
running_context
|
|
57
57
|
)
|
|
58
|
-
logger.debug(
|
|
58
|
+
Ductwork.logger.debug(
|
|
59
59
|
msg: "Creating new thread",
|
|
60
60
|
role: :job_worker_runner,
|
|
61
61
|
pipeline: pipeline
|
|
@@ -65,7 +65,7 @@ module Ductwork
|
|
|
65
65
|
end
|
|
66
66
|
thread.name = "ductwork.job_worker_#{i}"
|
|
67
67
|
|
|
68
|
-
logger.debug(
|
|
68
|
+
Ductwork.logger.debug(
|
|
69
69
|
msg: "Created new thread",
|
|
70
70
|
role: :job_worker_runner,
|
|
71
71
|
pipeline: pipeline
|
|
@@ -90,13 +90,13 @@ module Ductwork
|
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
def attempt_synchronize_threads
|
|
93
|
-
logger.debug(
|
|
93
|
+
Ductwork.logger.debug(
|
|
94
94
|
msg: "Attempting to synchronize threads",
|
|
95
95
|
role: :job_worker_runner,
|
|
96
96
|
pipeline: pipeline
|
|
97
97
|
)
|
|
98
98
|
threads.each { |thread| thread.join(0.1) }
|
|
99
|
-
logger.debug(
|
|
99
|
+
Ductwork.logger.debug(
|
|
100
100
|
msg: "Synchronizing threads timed out",
|
|
101
101
|
role: :job_worker_runner,
|
|
102
102
|
pipeline: pipeline
|
|
@@ -104,11 +104,11 @@ module Ductwork
|
|
|
104
104
|
end
|
|
105
105
|
|
|
106
106
|
def report_heartbeat!
|
|
107
|
-
logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
|
|
107
|
+
Ductwork.logger.debug(msg: "Reporting heartbeat", role: :job_worker_runner)
|
|
108
108
|
Ductwork.wrap_with_app_executor do
|
|
109
109
|
Ductwork::Process.report_heartbeat!
|
|
110
110
|
end
|
|
111
|
-
logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
|
|
111
|
+
Ductwork.logger.debug(msg: "Reported heartbeat", role: :job_worker_runner)
|
|
112
112
|
end
|
|
113
113
|
|
|
114
114
|
def shutdown!
|
|
@@ -122,7 +122,7 @@ module Ductwork
|
|
|
122
122
|
timeout = Ductwork.configuration.job_worker_shutdown_timeout
|
|
123
123
|
deadline = Time.current + timeout
|
|
124
124
|
|
|
125
|
-
logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
|
|
125
|
+
Ductwork.logger.debug(msg: "Attempting graceful shutdown", role: :job_worker_runner)
|
|
126
126
|
while Time.current < deadline && threads.any?(&:alive?)
|
|
127
127
|
threads.each do |thread|
|
|
128
128
|
break if Time.current < deadline
|
|
@@ -138,7 +138,7 @@ module Ductwork
|
|
|
138
138
|
threads.each do |thread|
|
|
139
139
|
if thread.alive?
|
|
140
140
|
thread.kill
|
|
141
|
-
logger.debug(
|
|
141
|
+
Ductwork.logger.debug(
|
|
142
142
|
msg: "Killed thread",
|
|
143
143
|
role: :job_worker_runner,
|
|
144
144
|
thread: thread.name
|
|
@@ -155,10 +155,6 @@ module Ductwork
|
|
|
155
155
|
).delete
|
|
156
156
|
end
|
|
157
157
|
end
|
|
158
|
-
|
|
159
|
-
def logger
|
|
160
|
-
Ductwork.configuration.logger
|
|
161
|
-
end
|
|
162
158
|
end
|
|
163
159
|
end
|
|
164
160
|
end
|
|
@@ -28,7 +28,7 @@ module Ductwork
|
|
|
28
28
|
.update_all(claimed_for_advancing_at: Time.current)
|
|
29
29
|
|
|
30
30
|
if rows_updated == 1
|
|
31
|
-
logger.debug(
|
|
31
|
+
Ductwork.logger.debug(
|
|
32
32
|
msg: "Pipeline claimed",
|
|
33
33
|
pipeline: klass,
|
|
34
34
|
role: :pipeline_advancer
|
|
@@ -37,13 +37,13 @@ module Ductwork
|
|
|
37
37
|
pipeline = Ductwork::Pipeline.find(id)
|
|
38
38
|
pipeline.advance!
|
|
39
39
|
|
|
40
|
-
logger.debug(
|
|
40
|
+
Ductwork.logger.debug(
|
|
41
41
|
msg: "Pipeline advanced",
|
|
42
42
|
pipeline: klass,
|
|
43
43
|
role: :pipeline_advancer
|
|
44
44
|
)
|
|
45
45
|
else
|
|
46
|
-
logger.debug(
|
|
46
|
+
Ductwork.logger.debug(
|
|
47
47
|
msg: "Did not claim pipeline, avoided race condition",
|
|
48
48
|
pipeline: klass,
|
|
49
49
|
role: :pipeline_advancer
|
|
@@ -57,7 +57,7 @@ module Ductwork
|
|
|
57
57
|
last_advanced_at: Time.current
|
|
58
58
|
)
|
|
59
59
|
else
|
|
60
|
-
logger.debug(
|
|
60
|
+
Ductwork.logger.debug(
|
|
61
61
|
msg: "No pipeline needs advancing",
|
|
62
62
|
pipeline: klass,
|
|
63
63
|
id: id,
|
|
@@ -82,10 +82,6 @@ module Ductwork
|
|
|
82
82
|
end
|
|
83
83
|
end
|
|
84
84
|
end
|
|
85
|
-
|
|
86
|
-
def logger
|
|
87
|
-
Ductwork.configuration.logger
|
|
88
|
-
end
|
|
89
85
|
end
|
|
90
86
|
end
|
|
91
87
|
end
|
|
@@ -25,7 +25,7 @@ module Ductwork
|
|
|
25
25
|
|
|
26
26
|
def run
|
|
27
27
|
create_process!
|
|
28
|
-
logger.debug(
|
|
28
|
+
Ductwork.logger.debug(
|
|
29
29
|
msg: "Entering main work loop",
|
|
30
30
|
role: :pipeline_advancer_runner
|
|
31
31
|
)
|
|
@@ -51,7 +51,7 @@ module Ductwork
|
|
|
51
51
|
klass
|
|
52
52
|
)
|
|
53
53
|
|
|
54
|
-
logger.debug(
|
|
54
|
+
Ductwork.logger.debug(
|
|
55
55
|
msg: "Creating new thread",
|
|
56
56
|
role: :pipeline_advancer_runner,
|
|
57
57
|
pipeline: klass
|
|
@@ -63,7 +63,7 @@ module Ductwork
|
|
|
63
63
|
end
|
|
64
64
|
thread.name = "ductwork.pipeline_advancer.#{klass}"
|
|
65
65
|
|
|
66
|
-
logger.debug(
|
|
66
|
+
Ductwork.logger.debug(
|
|
67
67
|
msg: "Created new thread",
|
|
68
68
|
role: :pipeline_advancer_runner,
|
|
69
69
|
thread: thread.name,
|
|
@@ -75,12 +75,12 @@ module Ductwork
|
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
def attempt_synchronize_threads
|
|
78
|
-
logger.debug(
|
|
78
|
+
Ductwork.logger.debug(
|
|
79
79
|
msg: "Attempting to synchronize threads",
|
|
80
80
|
role: :pipeline_advancer_runner
|
|
81
81
|
)
|
|
82
82
|
threads.each { |thread| thread.join(0.1) }
|
|
83
|
-
logger.debug(
|
|
83
|
+
Ductwork.logger.debug(
|
|
84
84
|
msg: "Synchronizing threads timed out",
|
|
85
85
|
role: :pipeline_advancer_runner
|
|
86
86
|
)
|
|
@@ -97,11 +97,11 @@ module Ductwork
|
|
|
97
97
|
end
|
|
98
98
|
|
|
99
99
|
def report_heartbeat!
|
|
100
|
-
logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
|
|
100
|
+
Ductwork.logger.debug(msg: "Reporting heartbeat", role: :pipeline_advancer_runner)
|
|
101
101
|
Ductwork.wrap_with_app_executor do
|
|
102
102
|
Ductwork::Process.report_heartbeat!
|
|
103
103
|
end
|
|
104
|
-
logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
|
|
104
|
+
Ductwork.logger.debug(msg: "Reported heartbeat", role: :pipeline_advancer_runner)
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
def shutdown
|
|
@@ -113,7 +113,7 @@ module Ductwork
|
|
|
113
113
|
end
|
|
114
114
|
|
|
115
115
|
def log_shutting_down
|
|
116
|
-
logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
|
|
116
|
+
Ductwork.logger.debug(msg: "Shutting down", role: :pipeline_advancer_runner)
|
|
117
117
|
end
|
|
118
118
|
|
|
119
119
|
def stop_running_context
|
|
@@ -124,7 +124,7 @@ module Ductwork
|
|
|
124
124
|
timeout = Ductwork.configuration.pipeline_shutdown_timeout
|
|
125
125
|
deadline = Time.current + timeout
|
|
126
126
|
|
|
127
|
-
logger.debug(
|
|
127
|
+
Ductwork.logger.debug(
|
|
128
128
|
msg: "Attempting graceful shutdown",
|
|
129
129
|
role: :pipeline_advancer_runner
|
|
130
130
|
)
|
|
@@ -143,7 +143,7 @@ module Ductwork
|
|
|
143
143
|
threads.each do |thread|
|
|
144
144
|
if thread.alive?
|
|
145
145
|
thread.kill
|
|
146
|
-
logger.debug(
|
|
146
|
+
Ductwork.logger.debug(
|
|
147
147
|
msg: "Killed thread",
|
|
148
148
|
role: :pipeline_advancer_runner,
|
|
149
149
|
thread: thread.name
|
|
@@ -160,10 +160,6 @@ module Ductwork
|
|
|
160
160
|
).delete
|
|
161
161
|
end
|
|
162
162
|
end
|
|
163
|
-
|
|
164
|
-
def logger
|
|
165
|
-
Ductwork.configuration.logger
|
|
166
|
-
end
|
|
167
163
|
end
|
|
168
164
|
end
|
|
169
165
|
end
|
|
@@ -22,14 +22,14 @@ module Ductwork
|
|
|
22
22
|
end
|
|
23
23
|
|
|
24
24
|
workers << { metadata:, pid:, block: }
|
|
25
|
-
logger.debug(
|
|
25
|
+
Ductwork.logger.debug(
|
|
26
26
|
msg: "Started child process (#{pid}) with metadata #{metadata}",
|
|
27
27
|
pid: pid
|
|
28
28
|
)
|
|
29
29
|
end
|
|
30
30
|
|
|
31
31
|
def run
|
|
32
|
-
logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
|
|
32
|
+
Ductwork.logger.debug(msg: "Entering main work loop", role: :supervisor, pid: ::Process.pid)
|
|
33
33
|
|
|
34
34
|
while running
|
|
35
35
|
sleep(Ductwork.configuration.supervisor_polling_timeout)
|
|
@@ -42,7 +42,7 @@ module Ductwork
|
|
|
42
42
|
def shutdown
|
|
43
43
|
@running = false
|
|
44
44
|
|
|
45
|
-
logger.debug(msg: "Beginning shutdown", role: :supervisor)
|
|
45
|
+
Ductwork.logger.debug(msg: "Beginning shutdown", role: :supervisor)
|
|
46
46
|
terminate_gracefully
|
|
47
47
|
wait_for_workers_to_exit
|
|
48
48
|
terminate_immediately
|
|
@@ -54,7 +54,7 @@ module Ductwork
|
|
|
54
54
|
attr_reader :running
|
|
55
55
|
|
|
56
56
|
def check_workers
|
|
57
|
-
logger.debug(msg: "Checking workers are alive", role: :supervisor)
|
|
57
|
+
Ductwork.logger.debug(msg: "Checking workers are alive", role: :supervisor)
|
|
58
58
|
|
|
59
59
|
workers.each do |worker|
|
|
60
60
|
if process_dead?(worker[:pid])
|
|
@@ -63,7 +63,7 @@ module Ductwork
|
|
|
63
63
|
worker[:block].call(worker[:metadata])
|
|
64
64
|
end
|
|
65
65
|
worker[:pid] = new_pid
|
|
66
|
-
logger.debug(
|
|
66
|
+
Ductwork.logger.debug(
|
|
67
67
|
msg: "Restarted process (#{old_pid}) as (#{new_pid})",
|
|
68
68
|
role: :supervisor,
|
|
69
69
|
old_pid: old_pid,
|
|
@@ -72,12 +72,12 @@ module Ductwork
|
|
|
72
72
|
end
|
|
73
73
|
end
|
|
74
74
|
|
|
75
|
-
logger.debug(msg: "All workers are alive or revived", role: :supervisor)
|
|
75
|
+
Ductwork.logger.debug(msg: "All workers are alive or revived", role: :supervisor)
|
|
76
76
|
end
|
|
77
77
|
|
|
78
78
|
def terminate_gracefully
|
|
79
79
|
workers.each do |worker|
|
|
80
|
-
logger.debug(
|
|
80
|
+
Ductwork.logger.debug(
|
|
81
81
|
msg: "Sending TERM signal to process (#{worker[:pid]})",
|
|
82
82
|
role: :supervisor,
|
|
83
83
|
pid: worker[:pid],
|
|
@@ -95,7 +95,7 @@ module Ductwork
|
|
|
95
95
|
workers.each_with_index do |worker, index|
|
|
96
96
|
if ::Process.wait(worker[:pid], ::Process::WNOHANG)
|
|
97
97
|
workers[index] = nil
|
|
98
|
-
logger.debug(
|
|
98
|
+
Ductwork.logger.debug(
|
|
99
99
|
msg: "Child process (#{worker[:pid]}) stopped successfully",
|
|
100
100
|
role: :supervisor,
|
|
101
101
|
pid: worker[:pid]
|
|
@@ -108,7 +108,7 @@ module Ductwork
|
|
|
108
108
|
|
|
109
109
|
def terminate_immediately
|
|
110
110
|
workers.each_with_index do |worker, index|
|
|
111
|
-
logger.debug(
|
|
111
|
+
Ductwork.logger.debug(
|
|
112
112
|
msg: "Sending KILL signal to process (#{worker[:pid]})",
|
|
113
113
|
role: :supervisor,
|
|
114
114
|
pid: worker[:pid],
|
|
@@ -117,7 +117,7 @@ module Ductwork
|
|
|
117
117
|
::Process.kill(:KILL, worker[:pid])
|
|
118
118
|
::Process.wait(worker[:pid])
|
|
119
119
|
workers[index] = nil
|
|
120
|
-
logger.debug(
|
|
120
|
+
Ductwork.logger.debug(
|
|
121
121
|
msg: "Child process (#{worker[:pid]}) killed after timeout",
|
|
122
122
|
role: :supervisor,
|
|
123
123
|
pid: worker[:pid]
|
|
@@ -151,10 +151,6 @@ module Ductwork
|
|
|
151
151
|
def now
|
|
152
152
|
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
|
153
153
|
end
|
|
154
|
-
|
|
155
|
-
def logger
|
|
156
|
-
Ductwork.configuration.logger
|
|
157
|
-
end
|
|
158
154
|
end
|
|
159
155
|
end
|
|
160
156
|
end
|
|
@@ -6,10 +6,9 @@ module Ductwork
|
|
|
6
6
|
def self.start!
|
|
7
7
|
supervisor = Ductwork::Processes::Supervisor.new
|
|
8
8
|
pipelines_to_advance = Ductwork.configuration.pipelines
|
|
9
|
-
logger = Ductwork.configuration.logger
|
|
10
9
|
|
|
11
10
|
supervisor.add_worker(metadata: { pipelines: pipelines_to_advance }) do
|
|
12
|
-
logger.debug(
|
|
11
|
+
Ductwork.logger.debug(
|
|
13
12
|
msg: "Starting Pipeline Advancer process",
|
|
14
13
|
role: :supervisor_runner
|
|
15
14
|
)
|
|
@@ -19,7 +18,7 @@ module Ductwork
|
|
|
19
18
|
|
|
20
19
|
pipelines_to_advance.each do |pipeline|
|
|
21
20
|
supervisor.add_worker(metadata: { pipeline: }) do
|
|
22
|
-
logger.debug(
|
|
21
|
+
Ductwork.logger.debug(
|
|
23
22
|
msg: "Starting Job Worker Runner process",
|
|
24
23
|
role: :supervisor_runner,
|
|
25
24
|
pipeline: pipeline
|
data/lib/ductwork/version.rb
CHANGED
data/lib/ductwork.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ductwork
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.3.
|
|
4
|
+
version: 0.3.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Tyler Ewing
|
|
@@ -86,7 +86,7 @@ dependencies:
|
|
|
86
86
|
description: Ductwork lets you build complex pipelines quickly and easily using intuitive
|
|
87
87
|
Ruby tooling and a natural DSL.
|
|
88
88
|
email:
|
|
89
|
-
-
|
|
89
|
+
- contact@getductwork.io
|
|
90
90
|
executables: []
|
|
91
91
|
extensions: []
|
|
92
92
|
extra_rdoc_files: []
|
|
@@ -140,7 +140,7 @@ licenses:
|
|
|
140
140
|
- LGPL-3.0-or-later
|
|
141
141
|
metadata:
|
|
142
142
|
homepage_uri: https://github.com/ductwork/ductwork
|
|
143
|
-
changelog_uri: https://github.com/ductwork/ductwork/blob/main/
|
|
143
|
+
changelog_uri: https://github.com/ductwork/ductwork/blob/main/CHANGELOG.md
|
|
144
144
|
rubygems_mfa_required: 'true'
|
|
145
145
|
documentation_uri: https://docs.getductwork.io/
|
|
146
146
|
rdoc_options: []
|