floe 0.20.1 → 0.21.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9620d0ace078bd773422b4a4a2f47c41891a312260f64a3a79c140aef0f9991
-  data.tar.gz: afd5e5e8c2f72845df2a0851714b77a08d2cd34dfdc081542b52bbeb63a90a23
+  metadata.gz: 17d64b12761527067ed26cda1737c6f4bd20c8d1c9affbb5edf78b176b4713c7
+  data.tar.gz: 22de6a27ac450b71d1467fb5b5fa304c6802e155f956bf2d0a72f6e097717aaa
 SHA512:
-  metadata.gz: 89130117cdb776734d58e28961a61e0c4fce39e465097e15200a907139bdd9c7597b05a0123099d8fdd5b186d349df6a0aacf971fc96cb436ffb415670c60a56
-  data.tar.gz: 152b98a4c2dae077b54a122dca7e2b138dc03c6b04809316c60709c53a550af109d5bf8cfef458a49882179360ef48d1427344bccf995517145f49a84e80245a
+  metadata.gz: 5e13a84b4568e4de952db7c05f5a29e6fc8fabaa0c944dc8b571a1514b5d9aae1118b94a4332e2e5456cdb0c776a629d1a19ef0d4a9803dd23e0844dc228cc03
+  data.tar.gz: 5512f1229704cb82a6e354388fc1873fe2eaf7b91d0410931a7e2e9baaadd5cf53815faaa3d4cee4cc4994d5ce8713b52a8ab451d241c3e5c857fc0bfc36f5a5

data/CHANGELOG.md

@@ -4,6 +4,11 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.21.0] - 2026-09-16
+### Added
+- Add volumes: option to run_async! for all container runners ([#347](https://github.com/ManageIQ/floe/pull/347))
+- Add entrypoint and command overrides to container runners ([#349](https://github.com/ManageIQ/floe/pull/349))
+
 ## [0.20.1] - 2026-09-15
 ### Fixed
 - Fix Execution Context not passed to Parallel branches ([#348](https://github.com/ManageIQ/floe/pull/348))
@@ -351,7 +356,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.20.1...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.21.0...HEAD
+[0.21.0]: https://github.com/ManageIQ/floe/compare/v0.20.1...v0.21.0
 [0.20.1]: https://github.com/ManageIQ/floe/compare/v0.20.0...v0.20.1
 [0.20.0]: https://github.com/ManageIQ/floe/compare/v0.19.2...v0.20.0
 [0.19.2]: https://github.com/ManageIQ/floe/compare/v0.19.1...v0.19.2

data/lib/floe/container_runner/docker.rb

@@ -18,8 +18,10 @@ module Floe
         @pull_policy = options["pull-policy"]
       end
 
-      def run_async!(resource, env, secrets, context)
+      def run_async!(resource, env, secrets, context, volumes: [], entrypoint: nil, command: nil)
         raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+        raise ArgumentError, "entrypoint must be a String" if entrypoint && !entrypoint.kind_of?(String)
+        raise ArgumentError, "command must be an Array"    if command && !command.kind_of?(Array)
 
         image          = resource.sub("docker://", "")
         execution_id   = context.execution["Id"]
@@ -30,7 +32,7 @@ module Floe
         end
 
         begin
-          runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"], context.logger)
+          runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"], context.logger, volumes, entrypoint, command)
           runner_context
         rescue AwesomeSpawn::CommandResultError => err
           cleanup(runner_context)
@@ -128,8 +130,8 @@ module Floe
 
       attr_reader :network
 
-      def run_container(image, env, execution_id, secrets_file, logger)
-        params = run_container_params(image, env, execution_id, secrets_file)
+      def run_container(image, env, execution_id, secrets_file, logger, volumes, entrypoint, command)
+        params = run_container_params(image, env, execution_id, secrets_file, volumes, entrypoint, command)
 
         logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
 
@@ -137,7 +139,7 @@ module Floe
         result.output.chomp
       end
 
-      def run_container_params(image, env, execution_id, secrets_file)
+      def run_container_params(image, env, execution_id, secrets_file, volumes, entrypoint, command)
         params  = ["run"]
         params << :detach
         params += env.map { |k, v| [:e, "#{k}=#{v}"] }
@@ -146,8 +148,20 @@ module Floe
         params << [:net, "host"] if @network == "host"
         params << [:label, "execution_id=#{execution_id}"]
         params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
+        params += volumes.map { |v| [:v, volume_to_flag(v)] }
+        params << [:entrypoint, entrypoint] if entrypoint
         params << [:name, container_name(image)]
         params << image
+        params.concat(command) if command
+        params
+      end
+
+      def volume_to_flag(volume)
+        options = volume.fetch(:options, "z")
+        source  = volume[:volume_name] || volume[:host_path]
+        flag    = "#{source}:#{volume[:container_path]}"
+        flag   += ":#{options}" if options && !options.empty?
+        flag
       end
 
       def wait_params(until_timestamp)

data/lib/floe/container_runner/kubernetes/host_path_volume_handler.rb

@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    class Kubernetes
+      # Mixin that handles host_path volumes for Kubernetes by staging the
+      # host-side directory to S3 for download by an init container.
+      module HostPathVolumeHandler
+        DEFAULT_INIT_IMAGE  = "curlimages/curl:latest"
+        INIT_CONTAINER_NAME = "floe-init"
+
+        def init_host_path_volume_options(options)
+          @s3_endpoint   = options["s3_endpoint"]
+          @s3_bucket     = options["s3_bucket"]
+          @s3_access_key = options["s3_access_key"]
+          @s3_secret_key = options["s3_secret_key"]
+          @init_image    = options.fetch("init_image", DEFAULT_INIT_IMAGE)
+        end
+
+        # Deletes every S3 object represented by staged_volumes, suppressing
+        # errors for each deletion.
+        def cleanup_staged_volumes(staged_volumes)
+          Array(staged_volumes).each do |volume|
+            key = volume[:s3_key] || volume["s3_key"]
+            Floe.logger.debug("Deleting s3://#{s3_bucket}/#{key}...")
+            delete_s3_object(key)
+            Floe.logger.debug("Deleting s3://#{s3_bucket}/#{key}...Complete")
+          end
+        end
+
+        private
+
+        attr_reader :s3_endpoint, :s3_bucket, :s3_access_key, :s3_secret_key, :init_image
+
+        def host_path_volume_instance_variables_to_hide
+          %i[@s3_access_key @s3_client @s3_secret_key]
+        end
+
+        # Upload each volume's host_path as a .tar.gz to S3, returning an array
+        # of hashes with :volume, :s3_key, :presigned_input_url.
+        def stage_host_path_volumes(volumes, execution_id, logger)
+          raise ArgumentError, "S3 must be configured (s3_endpoint, s3_bucket, s3_access_key, s3_secret_key) to use volumes with Kubernetes" unless s3_configured?
+
+          volumes.map do |volume|
+            s3_key = "floe/#{execution_id}/#{File.basename(volume[:container_path])}.tar.gz"
+            logger.debug("Staging volume #{volume[:host_path]} -> s3://#{s3_bucket}/#{s3_key}...")
+
+            tarball = create_tarball(volume[:host_path])
+            upload_to_s3(s3_key, tarball)
+            logger.debug("Staging volume #{volume[:host_path]} -> s3://#{s3_bucket}/#{s3_key}...Complete")
+            presigned_url = presign_s3_get(s3_key)
+
+            {:volume => volume, :s3_key => s3_key, :presigned_input_url => presigned_url}
+          end
+        end
+
+        def delete_s3_object(key)
+          s3_client.delete_object(:bucket => s3_bucket, :key => key)
+        rescue StandardError
+          nil
+        end
+
+        # Mutates spec in-place to add the emptyDir shared volumes and init
+        # container for staged volumes.
+        def add_host_path_volumes_to_spec!(spec, staged_volumes)
+          spec[:spec][:volumes] ||= []
+          primary = spec[:spec][:containers][0]
+          primary[:volumeMounts] ||= []
+
+          init_cmds          = []
+          init_volume_mounts = []
+
+          staged_volumes.each_with_index do |sv, idx|
+            volume     = sv[:volume]
+            share_name = "floe-volume-#{idx}"
+
+            # emptyDir shared between init container and primary
+            spec[:spec][:volumes] << {:name => share_name, :emptyDir => {}}
+
+            primary[:volumeMounts] << {
+              :name      => share_name,
+              :mountPath => volume[:container_path]
+            }
+
+            init_volume_mounts << {
+              :name      => share_name,
+              :mountPath => volume[:container_path]
+            }
+
+            # Init container downloads and unpacks each volume's tarball
+            init_cmds << "curl -fsSL '#{sv[:presigned_input_url]}' | tar -xzf - -C '#{volume[:container_path]}'"
+          end
+
+          # Init container: runs to completion before the primary starts, populating
+          # all shared volumes from S3. Primary command/args are left untouched.
+          spec[:spec][:initContainers] ||= []
+          spec[:spec][:initContainers] << {
+            :name         => INIT_CONTAINER_NAME,
+            :image        => init_image,
+            :command      => ["sh", "-c", init_cmds.join(" && ")],
+            :volumeMounts => init_volume_mounts
+          }
+        end
+
+        def s3_configured?
+          !!(s3_endpoint && s3_bucket && s3_access_key && s3_secret_key)
+        end
+
+        def s3_client
+          require "aws-sdk-s3"
+
+          @s3_client ||= Aws::S3::Client.new(
+            :endpoint          => s3_endpoint,
+            :region            => "us-east-1", # required by SDK even for non-AWS endpoints
+            :access_key_id     => s3_access_key,
+            :secret_access_key => s3_secret_key,
+            :force_path_style  => true         # required for MinIO and other non-AWS endpoints
+          )
+        end
+
+        def presign_s3_get(key)
+          require "aws-sdk-s3"
+
+          presigner = Aws::S3::Presigner.new(:client => s3_client)
+          presigner.presigned_url(:get_object, :bucket => s3_bucket, :key => key, :expires_in => 3600)
+        end
+
+        def create_tarball(source_path)
+          require "rubygems/package"
+          require "zlib"
+
+          gz_buffer = StringIO.new
+          Zlib::GzipWriter.wrap(gz_buffer) do |gz|
+            Gem::Package::TarWriter.new(gz) do |tar|
+              Dir.glob("#{source_path}/**/*", File::FNM_DOTMATCH).sort.each do |file|
+                relative = file.sub("#{source_path}/", "")
+                next if relative == "." || relative.empty?
+
+                stat = File.stat(file)
+                if stat.directory?
+                  tar.mkdir(relative, stat.mode)
+                else
+                  tar.add_file_simple(relative, stat.mode, stat.size) do |io|
+                    File.open(file, "rb") { |f| IO.copy_stream(f, io) }
+                  end
+                end
+              end
+            end
+          end
+          gz_buffer.string
+        end
+
+        def upload_to_s3(key, data)
+          s3_client.put_object(:bucket => s3_bucket, :key => key, :body => data)
+        end
+      end
+    end
+  end
+end

data/lib/floe/container_runner/kubernetes/persistent_volume_handler.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    class Kubernetes
+      # Mixin that attaches persistent volumes (e.g. pre-created docker volumes or
+      # PersistentVolumeClaims) directly to a pod spec without any staging.
+      # Each volume entry must have a :volume_name key (the PVC claim name) and
+      # a :container_path key. An optional :read_only key sets readOnly on the
+      # mount.
+      module PersistentVolumeHandler
+        private
+
+        # Mutates spec in-place to attach persistent volumes as PVCs. Each entry
+        # must have :volume_name (the PVC claim name) and :container_path.
+        def add_persistent_volumes_to_spec!(spec, persistent_volumes)
+          spec[:spec][:volumes] ||= []
+          primary = spec[:spec][:containers][0]
+          primary[:volumeMounts] ||= []
+
+          persistent_volumes.each_with_index do |vol, idx|
+            vol_name = "floe-persistent-volume-#{idx}"
+
+            spec[:spec][:volumes] << {
+              :name                  => vol_name,
+              :persistentVolumeClaim => {:claimName => vol[:volume_name]}
+            }
+
+            volume_mount = {
+              :name      => vol_name,
+              :mountPath => vol[:container_path]
+            }
+            volume_mount[:readOnly] = true if vol[:read_only]
+
+            primary[:volumeMounts] << volume_mount
+          end
+        end
+      end
+    end
+  end
+end

data/lib/floe/container_runner/kubernetes.rb

@@ -5,6 +5,12 @@ module Floe
     class Kubernetes < Floe::Runner
       include Floe::ContainerRunner::DockerMixin
 
+      require_relative "kubernetes/host_path_volume_handler"
+      include Floe::ContainerRunner::Kubernetes::HostPathVolumeHandler
+
+      require_relative "kubernetes/persistent_volume_handler"
+      include Floe::ContainerRunner::Kubernetes::PersistentVolumeHandler
+
       TOKEN_FILE      = "/run/secrets/kubernetes.io/serviceaccount/token"
       CA_CERT_FILE    = "/run/secrets/kubernetes.io/serviceaccount/ca.crt"
       RUNNING_PHASES  = %w[Pending Running].freeze
@@ -42,11 +48,15 @@ module Floe
         @pull_policy          = options["pull-policy"]
         @task_service_account = options["task_service_account"]
 
+        init_host_path_volume_options(options)
+
         super
       end
 
-      def run_async!(resource, env, secrets, context)
+      def run_async!(resource, env, secrets, context, volumes: [], entrypoint: nil, command: nil)
         raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+        raise ArgumentError, "entrypoint must be a String" if entrypoint && !entrypoint.kind_of?(String)
+        raise ArgumentError, "command must be an Array"    if command && !command.kind_of?(Array)
 
         image  = resource.sub("docker://", "")
         name   = container_name(image)
@@ -54,8 +64,17 @@ module Floe
         execution_id   = context.execution["Id"]
         runner_context = {"container_ref" => name, "container_state" => {"phase" => "Pending"}, "secrets_ref" => secret}
 
+        persistent_volumes, host_volumes = volumes.partition { |v| v[:volume_name] }
+        staged_volumes = stage_host_path_volumes(host_volumes, execution_id, context.logger) if host_volumes.any?
+        runner_context["staged_volumes"] = staged_volumes if staged_volumes
+
         begin
-          create_pod!(name, image, env, execution_id, secret)
+          spec = pod_spec(name, image, env, execution_id, secret, staged_volumes || [], persistent_volumes || [], entrypoint, command)
+          context.logger.debug("Running pod #{name} with image #{image}")
+          kubeclient.create_pod(spec)
+          # Always record the primary container name so get_pod_log targets the
+          # right container even when init containers are present.
+          runner_context["primary_container"] = spec.dig(:spec, :containers, 0, :name)
           runner_context
         rescue Kubeclient::HttpError => err
           cleanup(runner_context)
@@ -93,7 +112,9 @@ module Floe
           failed_state = failed_container_states(runner_context).first
           {"Error" => failed_state["reason"], "Cause" => failed_state["message"]}
         else
-          runner_context["output"] = kubeclient.get_pod_log(runner_context["container_ref"], namespace).body
+          log_options = {}
+          log_options[:container] = runner_context["primary_container"] if runner_context["primary_container"]
+          runner_context["output"] = kubeclient.get_pod_log(runner_context["container_ref"], namespace, **log_options).body
         end
       end
 
@@ -102,6 +123,8 @@ module Floe
 
         delete_pod(pod)       if pod
         delete_secret(secret) if secret
+
+        cleanup_staged_volumes(runner_context["staged_volumes"])
       end
 
       def wait(timeout: nil, events: %i[create update delete])
@@ -152,31 +175,25 @@ module Floe
         end
       end
 
+      def inspect
+        vars = instance_variables_to_inspect.map { |ivar| "#{ivar}=#{instance_variable_get(ivar).inspect}" }.join(", ")
+        prefix = Kernel.instance_method(:inspect).bind_call(self).split(' ', 2).first
+        "#{prefix} #{vars}>"
+      end
+
       private
 
       attr_reader :ca_file, :kubeconfig_file, :kubeconfig_context, :namespace, :server, :token, :verify_ssl
 
-      def pod_info(pod_name)
-        kubeclient.get_pod(pod_name, namespace)
-      rescue Kubeclient::HttpError => err
-        raise Floe::ExecutionError, "Failed to get status for pod #{namespace}/#{pod_name}: #{err}"
-      end
-
-      def pod_running?(context)
-        RUNNING_PHASES.include?(context.dig("container_state", "phase"))
-      end
-
-      def failed_container_states(context)
-        container_statuses = context.dig("container_state", "containerStatuses") || []
-        container_statuses.filter_map { |status| status["state"]&.values&.first }
-                          .select { |state| FAILURE_REASONS.include?(state["reason"]) }
+      def instance_variables_to_inspect
+        instance_variables - %i[@kubeclient @token] - host_path_volume_instance_variables_to_hide
       end
 
-      def container_failed?(context)
-        failed_container_states(context).any?
-      end
+      # ------------------------------------------------------------------
+      # Pod spec construction
+      # ------------------------------------------------------------------
 
-      def pod_spec(name, image, env, execution_id, secret = nil)
+      def pod_spec(name, image, env, execution_id, secret, staged_volumes, persistent_volumes, entrypoint, command)
         spec = {
           :kind       => "Pod",
           :apiVersion => "v1",
@@ -186,13 +203,7 @@ module Floe
             :labels    => {"execution_id" => execution_id}
           },
           :spec       => {
-            :containers    => [
-              {
-                :name  => name[0...-9], # remove the random suffix and its leading hyphen
-                :image => image,
-                :env   => env.map { |k, v| {:name => k, :value => v.to_s} }
-              }
-            ],
+            :containers    => [build_container_spec(name, image, env, entrypoint, command)],
             :restartPolicy => "Never"
           }
         }
@@ -222,11 +233,45 @@ module Floe
           ]
         end
 
+        add_host_path_volumes_to_spec!(spec, staged_volumes) if staged_volumes.any?
+        add_persistent_volumes_to_spec!(spec, persistent_volumes) if persistent_volumes.any?
+
         spec
       end
 
-      def create_pod!(name, image, env, execution_id, secret = nil)
-        kubeclient.create_pod(pod_spec(name, image, env, execution_id, secret))
+      def build_container_spec(name, image, env, entrypoint, command)
+        container = {
+          :name  => name[0...-9], # remove the random suffix and its leading hyphen
+          :image => image,
+          :env   => env.map { |k, v| {:name => k, :value => v.to_s} }
+        }
+        container[:command] = Array(entrypoint) if entrypoint
+        container[:args]    = command           if command
+        container
+      end
+
+      # ------------------------------------------------------------------
+      # Pod / secret lifecycle
+      # ------------------------------------------------------------------
+
+      def pod_info(pod_name)
+        kubeclient.get_pod(pod_name, namespace)
+      rescue Kubeclient::HttpError => err
+        raise Floe::ExecutionError, "Failed to get status for pod #{namespace}/#{pod_name}: #{err}"
+      end
+
+      def pod_running?(context)
+        RUNNING_PHASES.include?(context.dig("container_state", "phase"))
+      end
+
+      def failed_container_states(context)
+        container_statuses = context.dig("container_state", "containerStatuses") || []
+        container_statuses.filter_map { |status| status["state"]&.values&.first }
+                          .select { |state| FAILURE_REASONS.include?(state["reason"]) }
+      end
+
+      def container_failed?(context)
+        failed_container_states(context).any?
       end
 
       def delete_pod!(name)

data/lib/floe/container_runner/podman.rb

@@ -30,7 +30,7 @@ module Floe
 
       private
 
-      def run_container_params(image, env, execution_id, secret)
+      def run_container_params(image, env, execution_id, secret, volumes, entrypoint, command)
         params  = ["run"]
         params << :detach
         params += env.map { |k, v| [:e, "#{k}=#{v}"] }
@@ -39,8 +39,12 @@ module Floe
         params << [:net, "host"]        if @network == "host"
         params << [:label, "execution_id=#{execution_id}"]
         params << [:secret, secret] if secret
+        params += volumes.map { |v| [:v, volume_to_flag(v)] }
+        params << [:entrypoint, entrypoint] if entrypoint
         params << [:name, container_name(image)]
         params << image
+        params.concat(command) if command
+        params
       end
 
       def create_secret(secrets)

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.20.1"
+  VERSION = "0.21.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.21.0
 platform: ruby
 authors:
 - ManageIQ Developers
@@ -23,6 +23,20 @@ dependencies:
     - - ">"
       - !ruby/object:Gem::Version
         version: '5.2'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
   requirement: !ruby/object:Gem::Requirement
@@ -258,6 +272,8 @@ files:
 - lib/floe/container_runner/docker.rb
 - lib/floe/container_runner/docker_mixin.rb
 - lib/floe/container_runner/kubernetes.rb
+- lib/floe/container_runner/kubernetes/host_path_volume_handler.rb
+- lib/floe/container_runner/kubernetes/persistent_volume_handler.rb
 - lib/floe/container_runner/podman.rb
 - lib/floe/logging.rb
 - lib/floe/null_logger.rb