ductwork 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/README.md +1 -1
- data/lib/ductwork/cli.rb +9 -8
- data/lib/ductwork/configuration.rb +24 -4
- data/lib/ductwork/processes/job_worker.rb +2 -2
- data/lib/ductwork/processes/job_worker_runner.rb +20 -18
- data/lib/ductwork/processes/launcher.rb +45 -0
- data/lib/ductwork/processes/supervisor_runner.rb +13 -8
- data/lib/ductwork/version.rb +1 -1
- metadata +10 -9
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 4e9cbff3e15bf798c3ff2019f8b5637459b2b495f30c74e3b352bff97afa3b6b
|
|
4
|
+
data.tar.gz: 9d7a023b9c0018fe5aa8dc9291981146f0246deb6d8c1b98f658de146b2b78c5
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 2c019906fc749f7a0b47e3094823f3c867068bcabb0fea945ad47f017ac9ae39f659f5ca2b6a6741a666ad53eb0867720a0ce9ec2b57fac3e188b87ad6055dae
|
|
7
|
+
data.tar.gz: 1f584787c2f6588a86e59cbe6a2dac6fdaf014ff5b1b9632dc6816b1648d756267cdff77df7ffe06ba10ba74a060b72e0611a60f863d875a84981612319cc1a0
|
data/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,14 @@
|
|
|
1
1
|
# Ductwork Changelog
|
|
2
2
|
|
|
3
|
+
## [0.19.0]
|
|
4
|
+
|
|
5
|
+
- chore: bump rails-related dependencies to v8.1.2
|
|
6
|
+
- chore: remove ruby v3.2.9 from CI testing matrix - support is ending in March '26 but it's being removed now to better support edge rails
|
|
7
|
+
- feat: loosen rails version constraint to allow rails edge
|
|
8
|
+
- feat: respect "role" configuration when booting - ie. run main process as supervisor, pipeline advancer, or job worker
|
|
9
|
+
- feat: allow job worker runner to take a collection of pipelines and create workers for all configured pipelines - this will only happen when the "role" configuration is "worker" otherwise a process will be spun up for each pipeline
|
|
10
|
+
- feat: add "role" configuration - to be used to set the role of the entire ductwork running instance
|
|
11
|
+
|
|
3
12
|
## [0.18.0]
|
|
4
13
|
|
|
5
14
|
- fix: show countdown for pipelines/steps scheduled in the future
|
data/README.md
CHANGED
|
@@ -46,7 +46,7 @@ default: &default
|
|
|
46
46
|
pipelines: "*"
|
|
47
47
|
```
|
|
48
48
|
|
|
49
|
-
See the [Configuration Guide](https://docs.getductwork.io/
|
|
49
|
+
See the [Configuration Guide](https://docs.getductwork.io/getting-started/configuration.html) for all available options including thread counts, timeouts, and database settings.
|
|
50
50
|
|
|
51
51
|
## Usage
|
|
52
52
|
|
data/lib/ductwork/cli.rb
CHANGED
|
@@ -17,7 +17,7 @@ module Ductwork
|
|
|
17
17
|
option_parser.parse!(args)
|
|
18
18
|
auto_configure
|
|
19
19
|
puts banner
|
|
20
|
-
|
|
20
|
+
launch_processes
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
private
|
|
@@ -45,6 +45,7 @@ module Ductwork
|
|
|
45
45
|
end
|
|
46
46
|
|
|
47
47
|
def auto_configure
|
|
48
|
+
options[:role] = ENV.fetch("DUCTWORK_ROLE", nil)
|
|
48
49
|
Ductwork.configuration = Configuration.new(**options)
|
|
49
50
|
Ductwork.logger = if Ductwork.configuration.logger_source == "rails"
|
|
50
51
|
Rails.logger
|
|
@@ -63,17 +64,17 @@ module Ductwork
|
|
|
63
64
|
██║ ██║██║ ██║██║ ██║ ██║███╗██║██║ ██║██╔══██╗██╔═██╗
|
|
64
65
|
██████╔╝╚██████╔╝╚██████╗ ██║ ╚███╔███╔╝╚██████╔╝██║ ██║██║ ██╗
|
|
65
66
|
╚═════╝ ╚═════╝ ╚═════╝ ╚═╝ ╚══╝╚══╝ ╚═════╝ ╚═╝ ╚═╝╚═╝ ╚═╝
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
67
|
+
▒▒▓ ▒ ░▒▓▒ ▒ ▒ ░ ░▒ ▒ ░ ▒ ░░ ░ ▓░▒ ▒ ░ ▒░▒░▒░ ░ ▒▓ ░▒▓░▒ ▒▒ ▓▒
|
|
68
|
+
░ ▒ ▒ ░░▒░ ░ ░ ░ ▒ ░ ▒ ░ ░ ░ ▒ ▒░ ░▒ ░ ▒░░ ░▒ ▒░
|
|
69
|
+
░ ░ ░ ░░░ ░ ░ ░ ░ ░ ░ ░ ░ ░ ▒ ░░ ░ ░ ░░ ░
|
|
70
|
+
░ ░ ░ ░ ░ ░ ░ ░ ░ ░
|
|
71
|
+
░ ░
|
|
71
72
|
\e[0m
|
|
72
73
|
BANNER
|
|
73
74
|
end
|
|
74
75
|
|
|
75
|
-
def
|
|
76
|
-
Ductwork::Processes::
|
|
76
|
+
def launch_processes
|
|
77
|
+
Ductwork::Processes::Launcher.start_processes!
|
|
77
78
|
end
|
|
78
79
|
end
|
|
79
80
|
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
module Ductwork
|
|
4
|
-
class Configuration
|
|
4
|
+
class Configuration # rubocop:todo Metrics/ClassLength
|
|
5
5
|
DEFAULT_ENV = :default
|
|
6
6
|
DEFAULT_FILE_PATH = "config/ductwork.yml"
|
|
7
7
|
DEFAULT_JOB_WORKER_COUNT = 5 # threads
|
|
@@ -12,11 +12,15 @@ module Ductwork
|
|
|
12
12
|
DEFAULT_LOGGER_SOURCE = "default" # `Logger` instance writing to STDOUT
|
|
13
13
|
DEFAULT_PIPELINE_POLLING_TIMEOUT = 1 # second
|
|
14
14
|
DEFAULT_PIPELINE_SHUTDOWN_TIMEOUT = 20 # seconds
|
|
15
|
+
DEFAULT_ROLE = "all" # supervisor, pipeline advancer, and job workers
|
|
15
16
|
DEFAULT_STEPS_MAX_DEPTH = -1 # unlimited count
|
|
16
17
|
DEFAULT_SUPERVISOR_POLLING_TIMEOUT = 1 # second
|
|
17
18
|
DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT = 30 # seconds
|
|
18
19
|
DEFAULT_LOGGER = ::Logger.new($stdout)
|
|
19
20
|
PIPELINES_WILDCARD = "*"
|
|
21
|
+
VALID_ROLES = %w[all advancer worker].freeze
|
|
22
|
+
|
|
23
|
+
class InvalidRoleError < StandardError; end
|
|
20
24
|
|
|
21
25
|
attr_writer :job_worker_count, :job_worker_polling_timeout,
|
|
22
26
|
:job_worker_shutdown_timeout, :job_worker_max_retry,
|
|
@@ -25,13 +29,23 @@ module Ductwork
|
|
|
25
29
|
:steps_max_depth,
|
|
26
30
|
:supervisor_polling_timeout, :supervisor_shutdown_timeout
|
|
27
31
|
|
|
28
|
-
def initialize(path: DEFAULT_FILE_PATH)
|
|
32
|
+
def initialize(path: DEFAULT_FILE_PATH, role: nil)
|
|
29
33
|
full_path = Pathname.new(path)
|
|
30
34
|
data = ActiveSupport::ConfigurationFile.parse(full_path).deep_symbolize_keys
|
|
31
35
|
env = defined?(Rails) ? Rails.env.to_sym : DEFAULT_ENV
|
|
32
|
-
@config =
|
|
36
|
+
@config = if role.present?
|
|
37
|
+
data[env].merge(role:)
|
|
38
|
+
else
|
|
39
|
+
data[env]
|
|
40
|
+
end
|
|
33
41
|
rescue Errno::ENOENT
|
|
34
|
-
@config = {}
|
|
42
|
+
@config = { role: }.compact
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def role
|
|
46
|
+
r = config[:role] || DEFAULT_ROLE
|
|
47
|
+
validate_role!(r)
|
|
48
|
+
r
|
|
35
49
|
end
|
|
36
50
|
|
|
37
51
|
def pipelines
|
|
@@ -185,5 +199,11 @@ module Ductwork
|
|
|
185
199
|
config.dig(:supervisor, :shutdown_timeout) ||
|
|
186
200
|
DEFAULT_SUPERVISOR_SHUTDOWN_TIMEOUT
|
|
187
201
|
end
|
|
202
|
+
|
|
203
|
+
def validate_role!(role)
|
|
204
|
+
if VALID_ROLES.exclude?(role)
|
|
205
|
+
raise InvalidRoleError, "Must use a valid role"
|
|
206
|
+
end
|
|
207
|
+
end
|
|
188
208
|
end
|
|
189
209
|
end
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Ductwork
|
|
4
4
|
module Processes
|
|
5
5
|
class JobWorker
|
|
6
|
-
attr_reader :thread, :last_heartbeat_at, :job
|
|
6
|
+
attr_reader :thread, :last_heartbeat_at, :job, :pipeline
|
|
7
7
|
|
|
8
8
|
def initialize(pipeline, id)
|
|
9
9
|
@pipeline = pipeline
|
|
@@ -30,7 +30,7 @@ module Ductwork
|
|
|
30
30
|
|
|
31
31
|
private
|
|
32
32
|
|
|
33
|
-
attr_reader :
|
|
33
|
+
attr_reader :id, :running_context
|
|
34
34
|
|
|
35
35
|
def work_loop
|
|
36
36
|
run_hooks_for(:start)
|
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
module Ductwork
|
|
4
4
|
module Processes
|
|
5
5
|
class JobWorkerRunner
|
|
6
|
-
def initialize(
|
|
7
|
-
@
|
|
6
|
+
def initialize(*pipelines)
|
|
7
|
+
@pipelines = pipelines
|
|
8
8
|
@running_context = Ductwork::RunningContext.new
|
|
9
9
|
@job_workers = []
|
|
10
10
|
|
|
@@ -30,7 +30,7 @@ module Ductwork
|
|
|
30
30
|
Ductwork.logger.debug(
|
|
31
31
|
msg: "Entering main work loop",
|
|
32
32
|
role: :job_worker_runner,
|
|
33
|
-
|
|
33
|
+
pipelines: pipelines
|
|
34
34
|
)
|
|
35
35
|
|
|
36
36
|
while running?
|
|
@@ -45,7 +45,7 @@ module Ductwork
|
|
|
45
45
|
|
|
46
46
|
private
|
|
47
47
|
|
|
48
|
-
attr_reader :
|
|
48
|
+
attr_reader :pipelines, :running_context, :job_workers
|
|
49
49
|
|
|
50
50
|
def create_process_record!
|
|
51
51
|
Ductwork.wrap_with_app_executor do
|
|
@@ -58,16 +58,18 @@ module Ductwork
|
|
|
58
58
|
end
|
|
59
59
|
|
|
60
60
|
def start_job_workers
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
61
|
+
pipelines.each do |pipeline|
|
|
62
|
+
Ductwork.configuration.job_worker_count(pipeline).times do |i|
|
|
63
|
+
job_worker = Ductwork::Processes::JobWorker.new(pipeline, i)
|
|
64
|
+
job_workers.push(job_worker)
|
|
65
|
+
job_worker.start
|
|
66
|
+
|
|
67
|
+
Ductwork.logger.debug(
|
|
68
|
+
msg: "Created new job worker",
|
|
69
|
+
role: :job_worker_runner,
|
|
70
|
+
pipeline: pipeline
|
|
71
|
+
)
|
|
72
|
+
end
|
|
71
73
|
end
|
|
72
74
|
end
|
|
73
75
|
|
|
@@ -79,23 +81,23 @@ module Ductwork
|
|
|
79
81
|
Ductwork.logger.debug(
|
|
80
82
|
msg: "Checking thread health",
|
|
81
83
|
role: :job_worker_runner,
|
|
82
|
-
|
|
84
|
+
pipelines: pipelines
|
|
83
85
|
)
|
|
84
86
|
job_workers.each do |job_worker|
|
|
85
87
|
if !job_worker.alive?
|
|
86
88
|
job_worker.restart
|
|
87
89
|
|
|
88
90
|
Ductwork.logger.info(
|
|
89
|
-
msg: "Restarted
|
|
91
|
+
msg: "Restarted job worker",
|
|
90
92
|
role: :job_worker_runner,
|
|
91
|
-
pipeline: pipeline
|
|
93
|
+
pipeline: job_worker.pipeline
|
|
92
94
|
)
|
|
93
95
|
end
|
|
94
96
|
end
|
|
95
97
|
Ductwork.logger.debug(
|
|
96
98
|
msg: "Checked thread health",
|
|
97
99
|
role: :job_worker_runner,
|
|
98
|
-
|
|
100
|
+
pipelines: pipelines
|
|
99
101
|
)
|
|
100
102
|
end
|
|
101
103
|
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ductwork
|
|
4
|
+
module Processes
|
|
5
|
+
class Launcher
|
|
6
|
+
def self.start_processes!
|
|
7
|
+
new.start_processes!
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def initialize
|
|
11
|
+
@pipelines = Ductwork.configuration.pipelines
|
|
12
|
+
@runner_klass = case Ductwork.configuration.role
|
|
13
|
+
when "all"
|
|
14
|
+
supervisor_runner
|
|
15
|
+
when "advancer"
|
|
16
|
+
pipeline_advancer_runner
|
|
17
|
+
when "worker"
|
|
18
|
+
job_worker_runner
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def start_processes!
|
|
23
|
+
runner_klass
|
|
24
|
+
.new(*pipelines)
|
|
25
|
+
.run
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
attr_reader :pipelines, :runner_klass
|
|
31
|
+
|
|
32
|
+
def supervisor_runner
|
|
33
|
+
Ductwork::Processes::SupervisorRunner
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def pipeline_advancer_runner
|
|
37
|
+
Ductwork::Processes::PipelineAdvancerRunner
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def job_worker_runner
|
|
41
|
+
Ductwork::Processes::JobWorkerRunner
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -3,32 +3,37 @@
|
|
|
3
3
|
module Ductwork
|
|
4
4
|
module Processes
|
|
5
5
|
class SupervisorRunner
|
|
6
|
-
def
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
def initialize(*pipelines)
|
|
7
|
+
@pipelines = pipelines
|
|
8
|
+
@supervisor = Ductwork::Processes::Supervisor.new
|
|
9
|
+
end
|
|
9
10
|
|
|
10
|
-
|
|
11
|
+
def run
|
|
12
|
+
supervisor.add_worker(metadata: { pipelines: }) do
|
|
11
13
|
Ductwork.logger.debug(
|
|
12
14
|
msg: "Starting Pipeline Advancer process",
|
|
13
15
|
role: :supervisor_runner
|
|
14
16
|
)
|
|
15
|
-
Ductwork::Processes::PipelineAdvancerRunner
|
|
16
|
-
.new(*pipelines_to_advance).run
|
|
17
|
+
Ductwork::Processes::PipelineAdvancerRunner.new(*pipelines).run
|
|
17
18
|
end
|
|
18
19
|
|
|
19
|
-
|
|
20
|
+
pipelines.each do |pipeline|
|
|
20
21
|
supervisor.add_worker(metadata: { pipeline: }) do
|
|
21
22
|
Ductwork.logger.debug(
|
|
22
23
|
msg: "Starting Job Worker Runner process",
|
|
23
24
|
role: :supervisor_runner,
|
|
24
25
|
pipeline: pipeline
|
|
25
26
|
)
|
|
26
|
-
Ductwork::Processes::JobWorkerRunner.new(pipeline).run
|
|
27
|
+
Ductwork::Processes::JobWorkerRunner.new(*pipeline).run
|
|
27
28
|
end
|
|
28
29
|
end
|
|
29
30
|
|
|
30
31
|
supervisor.run
|
|
31
32
|
end
|
|
33
|
+
|
|
34
|
+
private
|
|
35
|
+
|
|
36
|
+
attr_reader :pipelines, :supervisor
|
|
32
37
|
end
|
|
33
38
|
end
|
|
34
39
|
end
|
data/lib/ductwork/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ductwork
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.19.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Tyler Ewing
|
|
@@ -18,7 +18,7 @@ dependencies:
|
|
|
18
18
|
version: '7.1'
|
|
19
19
|
- - "<"
|
|
20
20
|
- !ruby/object:Gem::Version
|
|
21
|
-
version: '8.
|
|
21
|
+
version: '8.3'
|
|
22
22
|
type: :runtime
|
|
23
23
|
prerelease: false
|
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -28,7 +28,7 @@ dependencies:
|
|
|
28
28
|
version: '7.1'
|
|
29
29
|
- - "<"
|
|
30
30
|
- !ruby/object:Gem::Version
|
|
31
|
-
version: '8.
|
|
31
|
+
version: '8.3'
|
|
32
32
|
- !ruby/object:Gem::Dependency
|
|
33
33
|
name: activerecord
|
|
34
34
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -38,7 +38,7 @@ dependencies:
|
|
|
38
38
|
version: '7.1'
|
|
39
39
|
- - "<"
|
|
40
40
|
- !ruby/object:Gem::Version
|
|
41
|
-
version: '8.
|
|
41
|
+
version: '8.3'
|
|
42
42
|
type: :runtime
|
|
43
43
|
prerelease: false
|
|
44
44
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -48,7 +48,7 @@ dependencies:
|
|
|
48
48
|
version: '7.1'
|
|
49
49
|
- - "<"
|
|
50
50
|
- !ruby/object:Gem::Version
|
|
51
|
-
version: '8.
|
|
51
|
+
version: '8.3'
|
|
52
52
|
- !ruby/object:Gem::Dependency
|
|
53
53
|
name: activesupport
|
|
54
54
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -58,7 +58,7 @@ dependencies:
|
|
|
58
58
|
version: '7.1'
|
|
59
59
|
- - "<"
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '8.
|
|
61
|
+
version: '8.3'
|
|
62
62
|
type: :runtime
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -68,7 +68,7 @@ dependencies:
|
|
|
68
68
|
version: '7.1'
|
|
69
69
|
- - "<"
|
|
70
70
|
- !ruby/object:Gem::Version
|
|
71
|
-
version: '8.
|
|
71
|
+
version: '8.3'
|
|
72
72
|
- !ruby/object:Gem::Dependency
|
|
73
73
|
name: railties
|
|
74
74
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -78,7 +78,7 @@ dependencies:
|
|
|
78
78
|
version: '7.1'
|
|
79
79
|
- - "<"
|
|
80
80
|
- !ruby/object:Gem::Version
|
|
81
|
-
version: '8.
|
|
81
|
+
version: '8.3'
|
|
82
82
|
type: :runtime
|
|
83
83
|
prerelease: false
|
|
84
84
|
version_requirements: !ruby/object:Gem::Requirement
|
|
@@ -88,7 +88,7 @@ dependencies:
|
|
|
88
88
|
version: '7.1'
|
|
89
89
|
- - "<"
|
|
90
90
|
- !ruby/object:Gem::Version
|
|
91
|
-
version: '8.
|
|
91
|
+
version: '8.3'
|
|
92
92
|
- !ruby/object:Gem::Dependency
|
|
93
93
|
name: zeitwerk
|
|
94
94
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -153,6 +153,7 @@ files:
|
|
|
153
153
|
- lib/ductwork/models/tuple.rb
|
|
154
154
|
- lib/ductwork/processes/job_worker.rb
|
|
155
155
|
- lib/ductwork/processes/job_worker_runner.rb
|
|
156
|
+
- lib/ductwork/processes/launcher.rb
|
|
156
157
|
- lib/ductwork/processes/pipeline_advancer.rb
|
|
157
158
|
- lib/ductwork/processes/pipeline_advancer_runner.rb
|
|
158
159
|
- lib/ductwork/processes/supervisor.rb
|