floe 0.20.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -1
- data/lib/floe/container_runner/docker.rb +21 -7
- data/lib/floe/container_runner/kubernetes/host_path_volume_handler.rb +159 -0
- data/lib/floe/container_runner/kubernetes/persistent_volume_handler.rb +41 -0
- data/lib/floe/container_runner/kubernetes.rb +75 -30
- data/lib/floe/container_runner/podman.rb +5 -1
- data/lib/floe/version.rb +1 -1
- data/lib/floe/workflow/context.rb +2 -1
- data/lib/floe/workflow/state.rb +12 -2
- data/lib/floe/workflow/states/task.rb +18 -19
- data/lib/floe/workflow.rb +20 -6
- metadata +17 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: bf92b02c87a1fa72f7a6424a6f01413fed9830a6e35194762095e7af6e76c366
|
|
4
|
+
data.tar.gz: 6d9b6209e73739fc7d6f7572b7e630366127de06eb2dca550542dc7558790ca6
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d03f3cde0036103469210a5a28eb20f2198c4cea7bb5306960a2fada722b109a2030f83858b51d9796a97cb30b6ce7a2f4b3be12209085e561009b9918d8a102
|
|
7
|
+
data.tar.gz: eca7b964bc9dbd5af65d122e01852650d4892731dc26fc30b71896bf256e98837d5d8c426ea84c59466aee6c4a8c383d67a455e8256c9f5b2fc5be97954065d6
|
data/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,20 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.22.0] - 2026-10-23
|
|
8
|
+
### Fixed
|
|
9
|
+
- Fix TypeError in Docker#wait when timeout is given ([#352](https://github.com/ManageIQ/floe/pull/352))
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
- Implement StateMachine TimeoutSeconds ([#350](https://github.com/ManageIQ/floe/pull/350))
|
|
13
|
+
- Add StateMachine Version ([#350](https://github.com/ManageIQ/floe/pull/350))
|
|
14
|
+
- Add timeout_at to Workflow and State ([#351](https://github.com/ManageIQ/floe/pull/351))
|
|
15
|
+
|
|
16
|
+
## [0.21.0] - 2026-09-16
|
|
17
|
+
### Added
|
|
18
|
+
- Add volumes: option to run_async! for all container runners ([#347](https://github.com/ManageIQ/floe/pull/347))
|
|
19
|
+
- Add entrypoint and command overrides to container runners ([#349](https://github.com/ManageIQ/floe/pull/349))
|
|
20
|
+
|
|
7
21
|
## [0.20.1] - 2026-09-15
|
|
8
22
|
### Fixed
|
|
9
23
|
- Fix Execution Context not passed to Parallel branches ([#348](https://github.com/ManageIQ/floe/pull/348))
|
|
@@ -351,7 +365,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
|
|
|
351
365
|
### Added
|
|
352
366
|
- Initial release
|
|
353
367
|
|
|
354
|
-
[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.
|
|
368
|
+
[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.22.0...HEAD
|
|
369
|
+
[0.22.0]: https://github.com/ManageIQ/floe/compare/v0.21.0...v0.22.0
|
|
370
|
+
[0.21.0]: https://github.com/ManageIQ/floe/compare/v0.20.1...v0.21.0
|
|
355
371
|
[0.20.1]: https://github.com/ManageIQ/floe/compare/v0.20.0...v0.20.1
|
|
356
372
|
[0.20.0]: https://github.com/ManageIQ/floe/compare/v0.19.2...v0.20.0
|
|
357
373
|
[0.19.2]: https://github.com/ManageIQ/floe/compare/v0.19.1...v0.19.2
|
|
@@ -18,8 +18,10 @@ module Floe
|
|
|
18
18
|
@pull_policy = options["pull-policy"]
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
def run_async!(resource, env, secrets, context)
|
|
21
|
+
def run_async!(resource, env, secrets, context, volumes: [], entrypoint: nil, command: nil)
|
|
22
22
|
raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
|
|
23
|
+
raise ArgumentError, "entrypoint must be a String" if entrypoint && !entrypoint.kind_of?(String)
|
|
24
|
+
raise ArgumentError, "command must be an Array" if command && !command.kind_of?(Array)
|
|
23
25
|
|
|
24
26
|
image = resource.sub("docker://", "")
|
|
25
27
|
execution_id = context.execution["Id"]
|
|
@@ -30,7 +32,7 @@ module Floe
|
|
|
30
32
|
end
|
|
31
33
|
|
|
32
34
|
begin
|
|
33
|
-
runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"], context.logger)
|
|
35
|
+
runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"], context.logger, volumes, entrypoint, command)
|
|
34
36
|
runner_context
|
|
35
37
|
rescue AwesomeSpawn::CommandResultError => err
|
|
36
38
|
cleanup(runner_context)
|
|
@@ -46,7 +48,7 @@ module Floe
|
|
|
46
48
|
end
|
|
47
49
|
|
|
48
50
|
def wait(timeout: nil, events: %i[create update delete], &block)
|
|
49
|
-
until_timestamp =
|
|
51
|
+
until_timestamp = Time.now.utc + timeout if timeout
|
|
50
52
|
|
|
51
53
|
r, w = IO.pipe
|
|
52
54
|
|
|
@@ -128,8 +130,8 @@ module Floe
|
|
|
128
130
|
|
|
129
131
|
attr_reader :network
|
|
130
132
|
|
|
131
|
-
def run_container(image, env, execution_id, secrets_file, logger)
|
|
132
|
-
params = run_container_params(image, env, execution_id, secrets_file)
|
|
133
|
+
def run_container(image, env, execution_id, secrets_file, logger, volumes, entrypoint, command)
|
|
134
|
+
params = run_container_params(image, env, execution_id, secrets_file, volumes, entrypoint, command)
|
|
133
135
|
|
|
134
136
|
logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
|
|
135
137
|
|
|
@@ -137,7 +139,7 @@ module Floe
|
|
|
137
139
|
result.output.chomp
|
|
138
140
|
end
|
|
139
141
|
|
|
140
|
-
def run_container_params(image, env, execution_id, secrets_file)
|
|
142
|
+
def run_container_params(image, env, execution_id, secrets_file, volumes, entrypoint, command)
|
|
141
143
|
params = ["run"]
|
|
142
144
|
params << :detach
|
|
143
145
|
params += env.map { |k, v| [:e, "#{k}=#{v}"] }
|
|
@@ -146,15 +148,27 @@ module Floe
|
|
|
146
148
|
params << [:net, "host"] if @network == "host"
|
|
147
149
|
params << [:label, "execution_id=#{execution_id}"]
|
|
148
150
|
params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
|
|
151
|
+
params += volumes.map { |v| [:v, volume_to_flag(v)] }
|
|
152
|
+
params << [:entrypoint, entrypoint] if entrypoint
|
|
149
153
|
params << [:name, container_name(image)]
|
|
150
154
|
params << image
|
|
155
|
+
params.concat(command) if command
|
|
156
|
+
params
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
def volume_to_flag(volume)
|
|
160
|
+
options = volume.fetch(:options, "z")
|
|
161
|
+
source = volume[:volume_name] || volume[:host_path]
|
|
162
|
+
flag = "#{source}:#{volume[:container_path]}"
|
|
163
|
+
flag += ":#{options}" if options && !options.empty?
|
|
164
|
+
flag
|
|
151
165
|
end
|
|
152
166
|
|
|
153
167
|
def wait_params(until_timestamp)
|
|
154
168
|
since_timestamp = Time.now.utc.to_i
|
|
155
169
|
|
|
156
170
|
params = ["events", "--format", "{{json .}}", "--filter", "type=container", "--since", since_timestamp.to_s]
|
|
157
|
-
params += ["--until", until_timestamp.to_s] if until_timestamp
|
|
171
|
+
params += ["--until", until_timestamp.to_i.to_s] if until_timestamp
|
|
158
172
|
params
|
|
159
173
|
end
|
|
160
174
|
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Floe
|
|
4
|
+
class ContainerRunner
|
|
5
|
+
class Kubernetes
|
|
6
|
+
# Mixin that handles host_path volumes for Kubernetes by staging the
|
|
7
|
+
# host-side directory to S3 for download by an init container.
|
|
8
|
+
module HostPathVolumeHandler
|
|
9
|
+
DEFAULT_INIT_IMAGE = "curlimages/curl:latest"
|
|
10
|
+
INIT_CONTAINER_NAME = "floe-init"
|
|
11
|
+
|
|
12
|
+
def init_host_path_volume_options(options)
|
|
13
|
+
@s3_endpoint = options["s3_endpoint"]
|
|
14
|
+
@s3_bucket = options["s3_bucket"]
|
|
15
|
+
@s3_access_key = options["s3_access_key"]
|
|
16
|
+
@s3_secret_key = options["s3_secret_key"]
|
|
17
|
+
@init_image = options.fetch("init_image", DEFAULT_INIT_IMAGE)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Deletes every S3 object represented by staged_volumes, suppressing
|
|
21
|
+
# errors for each deletion.
|
|
22
|
+
def cleanup_staged_volumes(staged_volumes)
|
|
23
|
+
Array(staged_volumes).each do |volume|
|
|
24
|
+
key = volume[:s3_key] || volume["s3_key"]
|
|
25
|
+
Floe.logger.debug("Deleting s3://#{s3_bucket}/#{key}...")
|
|
26
|
+
delete_s3_object(key)
|
|
27
|
+
Floe.logger.debug("Deleting s3://#{s3_bucket}/#{key}...Complete")
|
|
28
|
+
end
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
attr_reader :s3_endpoint, :s3_bucket, :s3_access_key, :s3_secret_key, :init_image
|
|
34
|
+
|
|
35
|
+
def host_path_volume_instance_variables_to_hide
|
|
36
|
+
%i[@s3_access_key @s3_client @s3_secret_key]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Upload each volume's host_path as a .tar.gz to S3, returning an array
|
|
40
|
+
# of hashes with :volume, :s3_key, :presigned_input_url.
|
|
41
|
+
def stage_host_path_volumes(volumes, execution_id, logger)
|
|
42
|
+
raise ArgumentError, "S3 must be configured (s3_endpoint, s3_bucket, s3_access_key, s3_secret_key) to use volumes with Kubernetes" unless s3_configured?
|
|
43
|
+
|
|
44
|
+
volumes.map do |volume|
|
|
45
|
+
s3_key = "floe/#{execution_id}/#{File.basename(volume[:container_path])}.tar.gz"
|
|
46
|
+
logger.debug("Staging volume #{volume[:host_path]} -> s3://#{s3_bucket}/#{s3_key}...")
|
|
47
|
+
|
|
48
|
+
tarball = create_tarball(volume[:host_path])
|
|
49
|
+
upload_to_s3(s3_key, tarball)
|
|
50
|
+
logger.debug("Staging volume #{volume[:host_path]} -> s3://#{s3_bucket}/#{s3_key}...Complete")
|
|
51
|
+
presigned_url = presign_s3_get(s3_key)
|
|
52
|
+
|
|
53
|
+
{:volume => volume, :s3_key => s3_key, :presigned_input_url => presigned_url}
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
def delete_s3_object(key)
|
|
58
|
+
s3_client.delete_object(:bucket => s3_bucket, :key => key)
|
|
59
|
+
rescue StandardError
|
|
60
|
+
nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Mutates spec in-place to add the emptyDir shared volumes and init
|
|
64
|
+
# container for staged volumes.
|
|
65
|
+
def add_host_path_volumes_to_spec!(spec, staged_volumes)
|
|
66
|
+
spec[:spec][:volumes] ||= []
|
|
67
|
+
primary = spec[:spec][:containers][0]
|
|
68
|
+
primary[:volumeMounts] ||= []
|
|
69
|
+
|
|
70
|
+
init_cmds = []
|
|
71
|
+
init_volume_mounts = []
|
|
72
|
+
|
|
73
|
+
staged_volumes.each_with_index do |sv, idx|
|
|
74
|
+
volume = sv[:volume]
|
|
75
|
+
share_name = "floe-volume-#{idx}"
|
|
76
|
+
|
|
77
|
+
# emptyDir shared between init container and primary
|
|
78
|
+
spec[:spec][:volumes] << {:name => share_name, :emptyDir => {}}
|
|
79
|
+
|
|
80
|
+
primary[:volumeMounts] << {
|
|
81
|
+
:name => share_name,
|
|
82
|
+
:mountPath => volume[:container_path]
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
init_volume_mounts << {
|
|
86
|
+
:name => share_name,
|
|
87
|
+
:mountPath => volume[:container_path]
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
# Init container downloads and unpacks each volume's tarball
|
|
91
|
+
init_cmds << "curl -fsSL '#{sv[:presigned_input_url]}' | tar -xzf - -C '#{volume[:container_path]}'"
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Init container: runs to completion before the primary starts, populating
|
|
95
|
+
# all shared volumes from S3. Primary command/args are left untouched.
|
|
96
|
+
spec[:spec][:initContainers] ||= []
|
|
97
|
+
spec[:spec][:initContainers] << {
|
|
98
|
+
:name => INIT_CONTAINER_NAME,
|
|
99
|
+
:image => init_image,
|
|
100
|
+
:command => ["sh", "-c", init_cmds.join(" && ")],
|
|
101
|
+
:volumeMounts => init_volume_mounts
|
|
102
|
+
}
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def s3_configured?
|
|
106
|
+
!!(s3_endpoint && s3_bucket && s3_access_key && s3_secret_key)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def s3_client
|
|
110
|
+
require "aws-sdk-s3"
|
|
111
|
+
|
|
112
|
+
@s3_client ||= Aws::S3::Client.new(
|
|
113
|
+
:endpoint => s3_endpoint,
|
|
114
|
+
:region => "us-east-1", # required by SDK even for non-AWS endpoints
|
|
115
|
+
:access_key_id => s3_access_key,
|
|
116
|
+
:secret_access_key => s3_secret_key,
|
|
117
|
+
:force_path_style => true # required for MinIO and other non-AWS endpoints
|
|
118
|
+
)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def presign_s3_get(key)
|
|
122
|
+
require "aws-sdk-s3"
|
|
123
|
+
|
|
124
|
+
presigner = Aws::S3::Presigner.new(:client => s3_client)
|
|
125
|
+
presigner.presigned_url(:get_object, :bucket => s3_bucket, :key => key, :expires_in => 3600)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def create_tarball(source_path)
|
|
129
|
+
require "rubygems/package"
|
|
130
|
+
require "zlib"
|
|
131
|
+
|
|
132
|
+
gz_buffer = StringIO.new
|
|
133
|
+
Zlib::GzipWriter.wrap(gz_buffer) do |gz|
|
|
134
|
+
Gem::Package::TarWriter.new(gz) do |tar|
|
|
135
|
+
Dir.glob("#{source_path}/**/*", File::FNM_DOTMATCH).sort.each do |file|
|
|
136
|
+
relative = file.sub("#{source_path}/", "")
|
|
137
|
+
next if relative == "." || relative.empty?
|
|
138
|
+
|
|
139
|
+
stat = File.stat(file)
|
|
140
|
+
if stat.directory?
|
|
141
|
+
tar.mkdir(relative, stat.mode)
|
|
142
|
+
else
|
|
143
|
+
tar.add_file_simple(relative, stat.mode, stat.size) do |io|
|
|
144
|
+
File.open(file, "rb") { |f| IO.copy_stream(f, io) }
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
gz_buffer.string
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def upload_to_s3(key, data)
|
|
154
|
+
s3_client.put_object(:bucket => s3_bucket, :key => key, :body => data)
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
end
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Floe
|
|
4
|
+
class ContainerRunner
|
|
5
|
+
class Kubernetes
|
|
6
|
+
# Mixin that attaches persistent volumes (e.g. pre-created docker volumes or
|
|
7
|
+
# PersistentVolumeClaims) directly to a pod spec without any staging.
|
|
8
|
+
# Each volume entry must have a :volume_name key (the PVC claim name) and
|
|
9
|
+
# a :container_path key. An optional :read_only key sets readOnly on the
|
|
10
|
+
# mount.
|
|
11
|
+
module PersistentVolumeHandler
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
# Mutates spec in-place to attach persistent volumes as PVCs. Each entry
|
|
15
|
+
# must have :volume_name (the PVC claim name) and :container_path.
|
|
16
|
+
def add_persistent_volumes_to_spec!(spec, persistent_volumes)
|
|
17
|
+
spec[:spec][:volumes] ||= []
|
|
18
|
+
primary = spec[:spec][:containers][0]
|
|
19
|
+
primary[:volumeMounts] ||= []
|
|
20
|
+
|
|
21
|
+
persistent_volumes.each_with_index do |vol, idx|
|
|
22
|
+
vol_name = "floe-persistent-volume-#{idx}"
|
|
23
|
+
|
|
24
|
+
spec[:spec][:volumes] << {
|
|
25
|
+
:name => vol_name,
|
|
26
|
+
:persistentVolumeClaim => {:claimName => vol[:volume_name]}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
volume_mount = {
|
|
30
|
+
:name => vol_name,
|
|
31
|
+
:mountPath => vol[:container_path]
|
|
32
|
+
}
|
|
33
|
+
volume_mount[:readOnly] = true if vol[:read_only]
|
|
34
|
+
|
|
35
|
+
primary[:volumeMounts] << volume_mount
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
@@ -5,6 +5,12 @@ module Floe
|
|
|
5
5
|
class Kubernetes < Floe::Runner
|
|
6
6
|
include Floe::ContainerRunner::DockerMixin
|
|
7
7
|
|
|
8
|
+
require_relative "kubernetes/host_path_volume_handler"
|
|
9
|
+
include Floe::ContainerRunner::Kubernetes::HostPathVolumeHandler
|
|
10
|
+
|
|
11
|
+
require_relative "kubernetes/persistent_volume_handler"
|
|
12
|
+
include Floe::ContainerRunner::Kubernetes::PersistentVolumeHandler
|
|
13
|
+
|
|
8
14
|
TOKEN_FILE = "/run/secrets/kubernetes.io/serviceaccount/token"
|
|
9
15
|
CA_CERT_FILE = "/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
10
16
|
RUNNING_PHASES = %w[Pending Running].freeze
|
|
@@ -42,11 +48,15 @@ module Floe
|
|
|
42
48
|
@pull_policy = options["pull-policy"]
|
|
43
49
|
@task_service_account = options["task_service_account"]
|
|
44
50
|
|
|
51
|
+
init_host_path_volume_options(options)
|
|
52
|
+
|
|
45
53
|
super
|
|
46
54
|
end
|
|
47
55
|
|
|
48
|
-
def run_async!(resource, env, secrets, context)
|
|
56
|
+
def run_async!(resource, env, secrets, context, volumes: [], entrypoint: nil, command: nil)
|
|
49
57
|
raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
|
|
58
|
+
raise ArgumentError, "entrypoint must be a String" if entrypoint && !entrypoint.kind_of?(String)
|
|
59
|
+
raise ArgumentError, "command must be an Array" if command && !command.kind_of?(Array)
|
|
50
60
|
|
|
51
61
|
image = resource.sub("docker://", "")
|
|
52
62
|
name = container_name(image)
|
|
@@ -54,8 +64,17 @@ module Floe
|
|
|
54
64
|
execution_id = context.execution["Id"]
|
|
55
65
|
runner_context = {"container_ref" => name, "container_state" => {"phase" => "Pending"}, "secrets_ref" => secret}
|
|
56
66
|
|
|
67
|
+
persistent_volumes, host_volumes = volumes.partition { |v| v[:volume_name] }
|
|
68
|
+
staged_volumes = stage_host_path_volumes(host_volumes, execution_id, context.logger) if host_volumes.any?
|
|
69
|
+
runner_context["staged_volumes"] = staged_volumes if staged_volumes
|
|
70
|
+
|
|
57
71
|
begin
|
|
58
|
-
|
|
72
|
+
spec = pod_spec(name, image, env, execution_id, secret, staged_volumes || [], persistent_volumes || [], entrypoint, command)
|
|
73
|
+
context.logger.debug("Running pod #{name} with image #{image}")
|
|
74
|
+
kubeclient.create_pod(spec)
|
|
75
|
+
# Always record the primary container name so get_pod_log targets the
|
|
76
|
+
# right container even when init containers are present.
|
|
77
|
+
runner_context["primary_container"] = spec.dig(:spec, :containers, 0, :name)
|
|
59
78
|
runner_context
|
|
60
79
|
rescue Kubeclient::HttpError => err
|
|
61
80
|
cleanup(runner_context)
|
|
@@ -93,7 +112,9 @@ module Floe
|
|
|
93
112
|
failed_state = failed_container_states(runner_context).first
|
|
94
113
|
{"Error" => failed_state["reason"], "Cause" => failed_state["message"]}
|
|
95
114
|
else
|
|
96
|
-
|
|
115
|
+
log_options = {}
|
|
116
|
+
log_options[:container] = runner_context["primary_container"] if runner_context["primary_container"]
|
|
117
|
+
runner_context["output"] = kubeclient.get_pod_log(runner_context["container_ref"], namespace, **log_options).body
|
|
97
118
|
end
|
|
98
119
|
end
|
|
99
120
|
|
|
@@ -102,6 +123,8 @@ module Floe
|
|
|
102
123
|
|
|
103
124
|
delete_pod(pod) if pod
|
|
104
125
|
delete_secret(secret) if secret
|
|
126
|
+
|
|
127
|
+
cleanup_staged_volumes(runner_context["staged_volumes"])
|
|
105
128
|
end
|
|
106
129
|
|
|
107
130
|
def wait(timeout: nil, events: %i[create update delete])
|
|
@@ -152,31 +175,25 @@ module Floe
|
|
|
152
175
|
end
|
|
153
176
|
end
|
|
154
177
|
|
|
178
|
+
def inspect
|
|
179
|
+
vars = instance_variables_to_inspect.map { |ivar| "#{ivar}=#{instance_variable_get(ivar).inspect}" }.join(", ")
|
|
180
|
+
prefix = Kernel.instance_method(:inspect).bind_call(self).split(' ', 2).first
|
|
181
|
+
"#{prefix} #{vars}>"
|
|
182
|
+
end
|
|
183
|
+
|
|
155
184
|
private
|
|
156
185
|
|
|
157
186
|
attr_reader :ca_file, :kubeconfig_file, :kubeconfig_context, :namespace, :server, :token, :verify_ssl
|
|
158
187
|
|
|
159
|
-
def
|
|
160
|
-
kubeclient
|
|
161
|
-
rescue Kubeclient::HttpError => err
|
|
162
|
-
raise Floe::ExecutionError, "Failed to get status for pod #{namespace}/#{pod_name}: #{err}"
|
|
163
|
-
end
|
|
164
|
-
|
|
165
|
-
def pod_running?(context)
|
|
166
|
-
RUNNING_PHASES.include?(context.dig("container_state", "phase"))
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
def failed_container_states(context)
|
|
170
|
-
container_statuses = context.dig("container_state", "containerStatuses") || []
|
|
171
|
-
container_statuses.filter_map { |status| status["state"]&.values&.first }
|
|
172
|
-
.select { |state| FAILURE_REASONS.include?(state["reason"]) }
|
|
188
|
+
def instance_variables_to_inspect
|
|
189
|
+
instance_variables - %i[@kubeclient @token] - host_path_volume_instance_variables_to_hide
|
|
173
190
|
end
|
|
174
191
|
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
192
|
+
# ------------------------------------------------------------------
|
|
193
|
+
# Pod spec construction
|
|
194
|
+
# ------------------------------------------------------------------
|
|
178
195
|
|
|
179
|
-
def pod_spec(name, image, env, execution_id, secret
|
|
196
|
+
def pod_spec(name, image, env, execution_id, secret, staged_volumes, persistent_volumes, entrypoint, command)
|
|
180
197
|
spec = {
|
|
181
198
|
:kind => "Pod",
|
|
182
199
|
:apiVersion => "v1",
|
|
@@ -186,13 +203,7 @@ module Floe
|
|
|
186
203
|
:labels => {"execution_id" => execution_id}
|
|
187
204
|
},
|
|
188
205
|
:spec => {
|
|
189
|
-
:containers => [
|
|
190
|
-
{
|
|
191
|
-
:name => name[0...-9], # remove the random suffix and its leading hyphen
|
|
192
|
-
:image => image,
|
|
193
|
-
:env => env.map { |k, v| {:name => k, :value => v.to_s} }
|
|
194
|
-
}
|
|
195
|
-
],
|
|
206
|
+
:containers => [build_container_spec(name, image, env, entrypoint, command)],
|
|
196
207
|
:restartPolicy => "Never"
|
|
197
208
|
}
|
|
198
209
|
}
|
|
@@ -222,11 +233,45 @@ module Floe
|
|
|
222
233
|
]
|
|
223
234
|
end
|
|
224
235
|
|
|
236
|
+
add_host_path_volumes_to_spec!(spec, staged_volumes) if staged_volumes.any?
|
|
237
|
+
add_persistent_volumes_to_spec!(spec, persistent_volumes) if persistent_volumes.any?
|
|
238
|
+
|
|
225
239
|
spec
|
|
226
240
|
end
|
|
227
241
|
|
|
228
|
-
def
|
|
229
|
-
|
|
242
|
+
def build_container_spec(name, image, env, entrypoint, command)
|
|
243
|
+
container = {
|
|
244
|
+
:name => name[0...-9], # remove the random suffix and its leading hyphen
|
|
245
|
+
:image => image,
|
|
246
|
+
:env => env.map { |k, v| {:name => k, :value => v.to_s} }
|
|
247
|
+
}
|
|
248
|
+
container[:command] = Array(entrypoint) if entrypoint
|
|
249
|
+
container[:args] = command if command
|
|
250
|
+
container
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# ------------------------------------------------------------------
|
|
254
|
+
# Pod / secret lifecycle
|
|
255
|
+
# ------------------------------------------------------------------
|
|
256
|
+
|
|
257
|
+
def pod_info(pod_name)
|
|
258
|
+
kubeclient.get_pod(pod_name, namespace)
|
|
259
|
+
rescue Kubeclient::HttpError => err
|
|
260
|
+
raise Floe::ExecutionError, "Failed to get status for pod #{namespace}/#{pod_name}: #{err}"
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
def pod_running?(context)
|
|
264
|
+
RUNNING_PHASES.include?(context.dig("container_state", "phase"))
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def failed_container_states(context)
|
|
268
|
+
container_statuses = context.dig("container_state", "containerStatuses") || []
|
|
269
|
+
container_statuses.filter_map { |status| status["state"]&.values&.first }
|
|
270
|
+
.select { |state| FAILURE_REASONS.include?(state["reason"]) }
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def container_failed?(context)
|
|
274
|
+
failed_container_states(context).any?
|
|
230
275
|
end
|
|
231
276
|
|
|
232
277
|
def delete_pod!(name)
|
|
@@ -30,7 +30,7 @@ module Floe
|
|
|
30
30
|
|
|
31
31
|
private
|
|
32
32
|
|
|
33
|
-
def run_container_params(image, env, execution_id, secret)
|
|
33
|
+
def run_container_params(image, env, execution_id, secret, volumes, entrypoint, command)
|
|
34
34
|
params = ["run"]
|
|
35
35
|
params << :detach
|
|
36
36
|
params += env.map { |k, v| [:e, "#{k}=#{v}"] }
|
|
@@ -39,8 +39,12 @@ module Floe
|
|
|
39
39
|
params << [:net, "host"] if @network == "host"
|
|
40
40
|
params << [:label, "execution_id=#{execution_id}"]
|
|
41
41
|
params << [:secret, secret] if secret
|
|
42
|
+
params += volumes.map { |v| [:v, volume_to_flag(v)] }
|
|
43
|
+
params << [:entrypoint, entrypoint] if entrypoint
|
|
42
44
|
params << [:name, container_name(image)]
|
|
43
45
|
params << image
|
|
46
|
+
params.concat(command) if command
|
|
47
|
+
params
|
|
44
48
|
end
|
|
45
49
|
|
|
46
50
|
def create_secret(secrets)
|
data/lib/floe/version.rb
CHANGED
|
@@ -26,7 +26,7 @@ module Floe
|
|
|
26
26
|
raise Floe::InvalidExecutionInput, "Invalid State Machine Execution Input: #{err}: was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
-
def prepare_start(start_at)
|
|
29
|
+
def prepare_start(start_at, timeout_seconds: nil)
|
|
30
30
|
return if started?
|
|
31
31
|
|
|
32
32
|
state["Name"] = start_at
|
|
@@ -35,6 +35,7 @@ module Floe
|
|
|
35
35
|
|
|
36
36
|
execution["Id"] ||= SecureRandom.uuid
|
|
37
37
|
execution["StartTime"] = Time.now.utc.iso8601
|
|
38
|
+
execution["TimeoutAt"] = (Time.now.utc + timeout_seconds).iso8601 if timeout_seconds
|
|
38
39
|
|
|
39
40
|
if logger.respond_to?(:execution_id=)
|
|
40
41
|
logger.execution_id = execution["Id"]
|
data/lib/floe/workflow/state.rb
CHANGED
|
@@ -45,8 +45,9 @@ module Floe
|
|
|
45
45
|
def run_nonblock!(context)
|
|
46
46
|
# Only start the state if it isn't already started and it isn't waiting
|
|
47
47
|
# from a prior Retry.
|
|
48
|
-
start(context)
|
|
49
|
-
|
|
48
|
+
start(context) unless started?(context) || waiting?(context)
|
|
49
|
+
raise Floe::TimeoutError if timed_out?(context)
|
|
50
|
+
return Errno::EAGAIN unless ready?(context)
|
|
50
51
|
|
|
51
52
|
finish(context)
|
|
52
53
|
rescue Floe::ExecutionError => e
|
|
@@ -115,6 +116,15 @@ module Floe
|
|
|
115
116
|
context.state["WaitUntil"] && Time.parse(context.state["WaitUntil"])
|
|
116
117
|
end
|
|
117
118
|
|
|
119
|
+
def timeout_at(context)
|
|
120
|
+
context.execution["TimeoutAt"] && Time.parse(context.execution["TimeoutAt"])
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
def timed_out?(context)
|
|
124
|
+
t = timeout_at(context)
|
|
125
|
+
t && Time.now.utc > t
|
|
126
|
+
end
|
|
127
|
+
|
|
118
128
|
def short_name
|
|
119
129
|
name.last
|
|
120
130
|
end
|
|
@@ -39,8 +39,7 @@ module Floe
|
|
|
39
39
|
def start(context)
|
|
40
40
|
super
|
|
41
41
|
|
|
42
|
-
|
|
43
|
-
wait_until!(context, :seconds => timeout_seconds) if timeout_seconds
|
|
42
|
+
set_timeout_at!(context)
|
|
44
43
|
|
|
45
44
|
input = process_input(context)
|
|
46
45
|
secrets = credentials&.value(context, context.input)
|
|
@@ -62,7 +61,6 @@ module Floe
|
|
|
62
61
|
end
|
|
63
62
|
|
|
64
63
|
def running?(context)
|
|
65
|
-
raise Floe::TimeoutError if timed_out?(context)
|
|
66
64
|
return false if finished?(context)
|
|
67
65
|
|
|
68
66
|
runner.status!(context.state["RunnerContext"])
|
|
@@ -80,8 +78,25 @@ module Floe
|
|
|
80
78
|
@end
|
|
81
79
|
end
|
|
82
80
|
|
|
81
|
+
def timeout_at(context)
|
|
82
|
+
workflow_timeout = super
|
|
83
|
+
task_timeout = context.state["TimeoutAt"] && Time.parse(context.state["TimeoutAt"])
|
|
84
|
+
|
|
85
|
+
[workflow_timeout, task_timeout].compact.min
|
|
86
|
+
end
|
|
87
|
+
|
|
83
88
|
private
|
|
84
89
|
|
|
90
|
+
def set_timeout_at!(context)
|
|
91
|
+
return if timeout_seconds.nil? && timeout_seconds_path.nil?
|
|
92
|
+
|
|
93
|
+
entered_time = Time.parse(context.state["EnteredTime"])
|
|
94
|
+
seconds = timeout_seconds || timeout_seconds_path.value(context, context.input)
|
|
95
|
+
raise Floe::PathError, "TimeoutSecondsPath references an invalid value [#{seconds}]" unless seconds.kind_of?(Integer) && seconds > 0
|
|
96
|
+
|
|
97
|
+
context.state["TimeoutAt"] = (entered_time + seconds).iso8601
|
|
98
|
+
end
|
|
99
|
+
|
|
85
100
|
attr_reader :runner
|
|
86
101
|
|
|
87
102
|
def validate_state!(workflow)
|
|
@@ -107,22 +122,6 @@ module Floe
|
|
|
107
122
|
runner.success?(context.state["RunnerContext"])
|
|
108
123
|
end
|
|
109
124
|
|
|
110
|
-
def timed_out?(context)
|
|
111
|
-
return false if timeout_seconds.nil? && timeout_seconds_path.nil?
|
|
112
|
-
|
|
113
|
-
timeout = timeout_seconds || timeout_seconds_path.value(context, context.input)
|
|
114
|
-
entered_time = Time.parse(context.state["EnteredTime"])
|
|
115
|
-
|
|
116
|
-
Time.now.utc > entered_time + timeout
|
|
117
|
-
end
|
|
118
|
-
|
|
119
|
-
def task_timed_out!(context)
|
|
120
|
-
context.state["RunnerContext"]["Error"] = "States.Timeout"
|
|
121
|
-
context.state["RunnerContext"]["Cause"] = "Task timed out"
|
|
122
|
-
|
|
123
|
-
false
|
|
124
|
-
end
|
|
125
|
-
|
|
126
125
|
def parse_error(output)
|
|
127
126
|
return if output.nil?
|
|
128
127
|
return output if output.kind_of?(Hash)
|
data/lib/floe/workflow.rb
CHANGED
|
@@ -42,9 +42,10 @@ module Floe
|
|
|
42
42
|
break if timeout && (timeout.zero? || Time.now.utc > run_until)
|
|
43
43
|
|
|
44
44
|
# Find the earliest time that we should wakeup if no container events
|
|
45
|
-
# are caught, either a workflow in a Wait or
|
|
46
|
-
# exceeded the requested timeout
|
|
45
|
+
# are caught, either a workflow in a Wait, Retry, or Timeout state or
|
|
46
|
+
# we've exceeded the requested timeout
|
|
47
47
|
wait_until = workflows.map(&:wait_until)
|
|
48
|
+
.concat(workflows.map(&:timeout_at))
|
|
48
49
|
.unshift(run_until)
|
|
49
50
|
.compact
|
|
50
51
|
.min
|
|
@@ -77,7 +78,7 @@ module Floe
|
|
|
77
78
|
end
|
|
78
79
|
end
|
|
79
80
|
|
|
80
|
-
attr_reader :comment, :context
|
|
81
|
+
attr_reader :comment, :context, :timeout_seconds, :version
|
|
81
82
|
|
|
82
83
|
def initialize(payload, context = nil, credentials = nil, name = nil)
|
|
83
84
|
payload = JSON.parse(payload) if payload.kind_of?(String)
|
|
@@ -88,8 +89,10 @@ module Floe
|
|
|
88
89
|
# caller should really put credentials into context and not pass that variable
|
|
89
90
|
context.credentials = credentials if credentials
|
|
90
91
|
|
|
91
|
-
@context
|
|
92
|
-
@comment
|
|
92
|
+
@context = context
|
|
93
|
+
@comment = payload["Comment"]
|
|
94
|
+
@timeout_seconds = payload["TimeoutSeconds"]
|
|
95
|
+
@version = payload["Version"] || "1.0"
|
|
93
96
|
|
|
94
97
|
super(payload, name)
|
|
95
98
|
rescue Floe::Error
|
|
@@ -136,6 +139,10 @@ module Floe
|
|
|
136
139
|
current_state.wait_until(context)
|
|
137
140
|
end
|
|
138
141
|
|
|
142
|
+
def timeout_at
|
|
143
|
+
current_state.timeout_at(context)
|
|
144
|
+
end
|
|
145
|
+
|
|
139
146
|
def status
|
|
140
147
|
context.status
|
|
141
148
|
end
|
|
@@ -150,7 +157,7 @@ module Floe
|
|
|
150
157
|
|
|
151
158
|
# setup a workflow
|
|
152
159
|
def start_workflow
|
|
153
|
-
context.prepare_start(start_at)
|
|
160
|
+
context.prepare_start(start_at, :timeout_seconds => timeout_seconds)
|
|
154
161
|
self
|
|
155
162
|
end
|
|
156
163
|
|
|
@@ -170,6 +177,13 @@ module Floe
|
|
|
170
177
|
|
|
171
178
|
private
|
|
172
179
|
|
|
180
|
+
def validate_workflow!
|
|
181
|
+
super
|
|
182
|
+
return unless timeout_seconds
|
|
183
|
+
|
|
184
|
+
invalid_field_error!("TimeoutSeconds", timeout_seconds, "must be a positive, non-zero integer") unless timeout_seconds.kind_of?(Integer) && timeout_seconds.positive?
|
|
185
|
+
end
|
|
186
|
+
|
|
173
187
|
def step!
|
|
174
188
|
next_state = {"Name" => context.next_state, "Guid" => SecureRandom.uuid, "PreviousStateGuid" => context.state["Guid"]}
|
|
175
189
|
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: floe
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.22.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- ManageIQ Developers
|
|
@@ -23,6 +23,20 @@ dependencies:
|
|
|
23
23
|
- - ">"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
25
|
version: '5.2'
|
|
26
|
+
- !ruby/object:Gem::Dependency
|
|
27
|
+
name: aws-sdk-s3
|
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
|
29
|
+
requirements:
|
|
30
|
+
- - "~>"
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: '1.0'
|
|
33
|
+
type: :runtime
|
|
34
|
+
prerelease: false
|
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '1.0'
|
|
26
40
|
- !ruby/object:Gem::Dependency
|
|
27
41
|
name: awesome_spawn
|
|
28
42
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -258,6 +272,8 @@ files:
|
|
|
258
272
|
- lib/floe/container_runner/docker.rb
|
|
259
273
|
- lib/floe/container_runner/docker_mixin.rb
|
|
260
274
|
- lib/floe/container_runner/kubernetes.rb
|
|
275
|
+
- lib/floe/container_runner/kubernetes/host_path_volume_handler.rb
|
|
276
|
+
- lib/floe/container_runner/kubernetes/persistent_volume_handler.rb
|
|
261
277
|
- lib/floe/container_runner/podman.rb
|
|
262
278
|
- lib/floe/logging.rb
|
|
263
279
|
- lib/floe/null_logger.rb
|