floe 0.10.0 → 0.11.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-04-05 00:00:00.000000000 Z
+date: 2024-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -185,8 +185,14 @@ files:
 - exe/floe
 - floe.gemspec
 - lib/floe.rb
+- lib/floe/container_runner.rb
+- lib/floe/container_runner/docker.rb
+- lib/floe/container_runner/docker_mixin.rb
+- lib/floe/container_runner/kubernetes.rb
+- lib/floe/container_runner/podman.rb
 - lib/floe/logging.rb
 - lib/floe/null_logger.rb
+- lib/floe/runner.rb
 - lib/floe/version.rb
 - lib/floe/workflow.rb
 - lib/floe/workflow/catcher.rb
@@ -200,11 +206,6 @@ files:
 - lib/floe/workflow/payload_template.rb
 - lib/floe/workflow/reference_path.rb
 - lib/floe/workflow/retrier.rb
-- lib/floe/workflow/runner.rb
-- lib/floe/workflow/runner/docker.rb
-- lib/floe/workflow/runner/docker_mixin.rb
-- lib/floe/workflow/runner/kubernetes.rb
-- lib/floe/workflow/runner/podman.rb
 - lib/floe/workflow/state.rb
 - lib/floe/workflow/states/choice.rb
 - lib/floe/workflow/states/fail.rb

data/lib/floe/workflow/runner/docker.rb

@@ -1,227 +0,0 @@
-# frozen_string_literal: true
-
-module Floe
-  class Workflow
-    class Runner
-      class Docker < Floe::Workflow::Runner
-        include DockerMixin
-
-        DOCKER_COMMAND = "docker"
-
-        def initialize(options = {})
-          require "awesome_spawn"
-          require "io/wait"
-          require "tempfile"
-
-          super
-
-          @network     = options.fetch("network", "bridge")
-          @pull_policy = options["pull-policy"]
-        end
-
-        def run_async!(resource, env = {}, secrets = {})
-          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
-
-          image = resource.sub("docker://", "")
-
-          runner_context = {}
-
-          if secrets && !secrets.empty?
-            runner_context["secrets_ref"] = create_secret(secrets)
-          end
-
-          begin
-            runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"])
-            runner_context
-          rescue AwesomeSpawn::CommandResultError => err
-            cleanup(runner_context)
-            {"Error" => "States.TaskFailed", "Cause" => err.to_s}
-          end
-        end
-
-        def cleanup(runner_context)
-          container_id, secrets_file = runner_context.values_at("container_ref", "secrets_ref")
-
-          delete_container(container_id) if container_id
-          delete_secret(secrets_file)    if secrets_file
-        end
-
-        def wait(timeout: nil, events: %i[create update delete], &block)
-          until_timestamp = Time.now.utc + timeout if timeout
-
-          r, w = IO.pipe
-
-          pid = AwesomeSpawn.run_detached(
-            self.class::DOCKER_COMMAND, :err => :out, :out => w, :params => wait_params(until_timestamp)
-          )
-
-          w.close
-
-          loop do
-            readable_timeout = until_timestamp - Time.now.utc if until_timestamp
-
-            # Wait for our end of the pipe to be readable and if it didn't timeout
-            # get the events from stdout
-            next if r.wait_readable(readable_timeout).nil?
-
-            # Get all events while the pipe is readable
-            notices = []
-            while r.ready?
-              notice = r.gets
-
-              # If the process has exited `r.gets` returns `nil` and the pipe is
-              # always `ready?`
-              break if notice.nil?
-
-              event, runner_context = parse_notice(notice)
-              next if event.nil? || !events.include?(event)
-
-              notices << [event, runner_context]
-            end
-
-            # If we're given a block yield the events otherwise return them
-            if block
-              notices.each(&block)
-            else
-              # Terminate the `docker events` process before returning the events
-              sigterm(pid)
-
-              return notices
-            end
-
-            # Check that the `docker events` process is still alive
-            Process.kill(0, pid)
-          rescue Errno::ESRCH
-            # Break out of the loop if the `docker events` process has exited
-            break
-          end
-        ensure
-          r.close
-        end
-
-        def status!(runner_context)
-          return if runner_context.key?("Error")
-
-          runner_context["container_state"] = inspect_container(runner_context["container_ref"])&.dig("State")
-        end
-
-        def running?(runner_context)
-          !!runner_context.dig("container_state", "Running")
-        end
-
-        def success?(runner_context)
-          runner_context.dig("container_state", "ExitCode") == 0
-        end
-
-        def output(runner_context)
-          return runner_context.slice("Error", "Cause") if runner_context.key?("Error")
-
-          output = docker!("logs", runner_context["container_ref"], :combined_output => true).output
-          runner_context["output"] = output
-        end
-
-        private
-
-        attr_reader :network
-
-        def run_container(image, env, secrets_file)
-          params = run_container_params(image, env, secrets_file)
-
-          logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
-
-          result = docker!(*params)
-          result.output
-        end
-
-        def run_container_params(image, env, secrets_file)
-          params  = ["run"]
-          params << :detach
-          params += env.map { |k, v| [:e, "#{k}=#{v}"] }
-          params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
-          params << [:pull, @pull_policy] if @pull_policy
-          params << [:net, "host"] if @network == "host"
-          params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
-          params << [:name, container_name(image)]
-          params << image
-        end
-
-        def wait_params(until_timestamp)
-          params = ["events", [:format, "{{json .}}"], [:filter, "type=container"], [:since, Time.now.utc.to_i]]
-          params << [:until, until_timestamp.to_i] if until_timestamp
-          params
-        end
-
-        def parse_notice(notice)
-          notice = JSON.parse(notice)
-
-          status  = notice["status"]
-          event   = docker_event_status_to_event(status)
-          running = event != :delete
-
-          name, exit_code = notice.dig("Actor", "Attributes")&.values_at("name", "exitCode")
-
-          runner_context = {"container_ref" => name, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
-
-          [event, runner_context]
-        rescue JSON::ParserError
-          []
-        end
-
-        def docker_event_status_to_event(status)
-          case status
-          when "create"
-            :create
-          when "start"
-            :update
-          when "die", "destroy"
-            :delete
-          else
-            :unkonwn
-          end
-        end
-
-        def inspect_container(container_id)
-          JSON.parse(docker!("inspect", container_id).output).first
-        rescue
-          nil
-        end
-
-        def delete_container(container_id)
-          docker!("rm", container_id)
-        rescue
-          nil
-        end
-
-        def delete_secret(secrets_file)
-          return unless File.exist?(secrets_file)
-
-          File.unlink(secrets_file)
-        rescue
-          nil
-        end
-
-        def create_secret(secrets)
-          secrets_file = Tempfile.new
-          secrets_file.write(secrets.to_json)
-          secrets_file.close
-          secrets_file.path
-        end
-
-        def sigterm(pid)
-          Process.kill("TERM", pid)
-        rescue Errno::ESRCH
-          nil
-        end
-
-        def global_docker_options
-          []
-        end
-
-        def docker!(*args, **kwargs)
-          params = global_docker_options + args
-          AwesomeSpawn.run!(self.class::DOCKER_COMMAND, :params => params, **kwargs)
-        end
-      end
-    end
-  end
-end

data/lib/floe/workflow/runner/docker_mixin.rb

@@ -1,32 +0,0 @@
-module Floe
-  class Workflow
-    class Runner
-      module DockerMixin
-        def image_name(image)
-          image.match(%r{^(?<repository>.+/)?(?<image>.+):(?<tag>.+)$})&.named_captures&.dig("image")
-        end
-
-        # 63 is the max kubernetes pod name length
-        # -5 for the "floe-" prefix
-        # -9 for the random hex suffix and leading hyphen
-        MAX_CONTAINER_NAME_SIZE = 63 - 5 - 9
-
-        def container_name(image)
-          name = image_name(image)
-          raise ArgumentError, "Invalid docker image [#{image}]" if name.nil?
-
-          # Normalize the image name to be used in the container name.
-          # This follows RFC 1123 Label names in Kubernetes as they are the most restrictive
-          # See https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
-          # and https://github.com/kubernetes/kubernetes/blob/952a9cb0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L178-L184
-          #
-          # This does not follow the leading and trailing character restriction because we will embed it
-          # below with a prefix and suffix that already conform to the RFC.
-          normalized_name = name.downcase.gsub(/[^a-z0-9-]/, "-")[0, MAX_CONTAINER_NAME_SIZE]
-
-          "floe-#{normalized_name}-#{SecureRandom.hex(4)}"
-        end
-      end
-    end
-  end
-end

data/lib/floe/workflow/runner/kubernetes.rb

@@ -1,331 +0,0 @@
-# frozen_string_literal: true
-
-module Floe
-  class Workflow
-    class Runner
-      class Kubernetes < Floe::Workflow::Runner
-        include DockerMixin
-
-        TOKEN_FILE      = "/run/secrets/kubernetes.io/serviceaccount/token"
-        CA_CERT_FILE    = "/run/secrets/kubernetes.io/serviceaccount/ca.crt"
-        RUNNING_PHASES  = %w[Pending Running].freeze
-        FAILURE_REASONS = %w[CrashLoopBackOff ImagePullBackOff ErrImagePull].freeze
-
-        def initialize(options = {})
-          require "active_support/core_ext/hash/keys"
-          require "awesome_spawn"
-          require "securerandom"
-          require "base64"
-          require "kubeclient"
-          require "yaml"
-
-          @kubeconfig_file    = ENV.fetch("KUBECONFIG", nil) || options.fetch("kubeconfig", File.join(Dir.home, ".kube", "config"))
-          @kubeconfig_context = options["kubeconfig_context"]
-
-          @token   = options["token"]
-          @token ||= File.read(options["token_file"]) if options.key?("token_file")
-          @token ||= File.read(TOKEN_FILE) if File.exist?(TOKEN_FILE)
-
-          @server   = options["server"]
-          @server ||= URI::HTTPS.build(:host => ENV.fetch("KUBERNETES_SERVICE_HOST"), :port => ENV.fetch("KUBERNETES_SERVICE_PORT", 6443)) if ENV.key?("KUBERNETES_SERVICE_HOST")
-
-          @ca_file   = options["ca_file"]
-          @ca_file ||= CA_CERT_FILE if File.exist?(CA_CERT_FILE)
-
-          @verify_ssl = options["verify_ssl"] == "false" ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
-
-          if server.nil? && token.nil? && !File.exist?(kubeconfig_file)
-            raise ArgumentError, "Missing connections options, provide a kubeconfig file or pass server and token via --docker-runner-options"
-          end
-
-          @namespace = options.fetch("namespace", "default")
-
-          @pull_policy          = options["pull-policy"]
-          @task_service_account = options["task_service_account"]
-
-          super
-        end
-
-        def run_async!(resource, env = {}, secrets = {})
-          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
-
-          image  = resource.sub("docker://", "")
-          name   = container_name(image)
-          secret = create_secret!(secrets) if secrets && !secrets.empty?
-
-          runner_context = {"container_ref" => name, "container_state" => {"phase" => "Pending"}, "secrets_ref" => secret}
-
-          begin
-            create_pod!(name, image, env, secret)
-            runner_context
-          rescue Kubeclient::HttpError => err
-            cleanup(runner_context)
-            {"Error" => "States.TaskFailed", "Cause" => err.to_s}
-          end
-        end
-
-        def status!(runner_context)
-          return if runner_context.key?("Error")
-
-          runner_context["container_state"] = pod_info(runner_context["container_ref"]).to_h.deep_stringify_keys["status"]
-        end
-
-        def running?(runner_context)
-          return false unless pod_running?(runner_context)
-          # If a pod is Pending and the containers are waiting with a failure
-          # reason such as ImagePullBackOff or CrashLoopBackOff then the pod
-          # will never be run.
-          return false if container_failed?(runner_context)
-
-          true
-        end
-
-        def success?(runner_context)
-          runner_context.dig("container_state", "phase") == "Succeeded"
-        end
-
-        def output(runner_context)
-          if runner_context.key?("Error")
-            runner_context.slice("Error", "Cause")
-          elsif container_failed?(runner_context)
-            failed_state = failed_container_states(runner_context).first
-            {"Error" => failed_state["reason"], "Cause" => failed_state["message"]}
-          else
-            runner_context["output"] = kubeclient.get_pod_log(runner_context["container_ref"], namespace).body
-          end
-        end
-
-        def cleanup(runner_context)
-          pod, secret = runner_context.values_at("container_ref", "secrets_ref")
-
-          delete_pod(pod)       if pod
-          delete_secret(secret) if secret
-        end
-
-        def wait(timeout: nil, events: %i[create update delete])
-          retry_connection = true
-
-          begin
-            watcher = kubeclient.watch_pods(:namespace => namespace)
-
-            retry_connection = true
-
-            if timeout.to_i > 0
-              timeout_thread = Thread.new do
-                sleep(timeout)
-                watcher.finish
-              end
-            end
-
-            watcher.each do |notice|
-              break if error_notice?(notice)
-
-              event = kube_notice_type_to_event(notice.type)
-              next unless events.include?(event)
-
-              runner_context = parse_notice(notice)
-              next if runner_context.nil?
-
-              if block_given?
-                yield [event, runner_context]
-              else
-                timeout_thread&.kill # If we break out before the timeout, kill the timeout thread
-                return [[event, runner_context]]
-              end
-            end
-          rescue Kubeclient::HttpError => err
-            raise unless err.error_code == 401 && retry_connection
-
-            @kubeclient = nil
-            retry_connection = false
-            retry
-          ensure
-            begin
-              watch&.finish
-            rescue
-              nil
-            end
-
-            timeout_thread&.join(0)
-          end
-        end
-
-        private
-
-        attr_reader :ca_file, :kubeconfig_file, :kubeconfig_context, :namespace, :server, :token, :verify_ssl
-
-        def pod_info(pod_name)
-          kubeclient.get_pod(pod_name, namespace)
-        end
-
-        def pod_running?(context)
-          RUNNING_PHASES.include?(context.dig("container_state", "phase"))
-        end
-
-        def failed_container_states(context)
-          container_statuses = context.dig("container_state", "containerStatuses") || []
-          container_statuses.filter_map { |status| status["state"]&.values&.first }
-                            .select { |state| FAILURE_REASONS.include?(state["reason"]) }
-        end
-
-        def container_failed?(context)
-          failed_container_states(context).any?
-        end
-
-        def pod_spec(name, image, env, secret = nil)
-          spec = {
-            :kind       => "Pod",
-            :apiVersion => "v1",
-            :metadata   => {
-              :name      => name,
-              :namespace => namespace
-            },
-            :spec       => {
-              :containers    => [
-                {
-                  :name  => name[0...-9], # remove the random suffix and its leading hyphen
-                  :image => image,
-                  :env   => env.map { |k, v| {:name => k, :value => v.to_s} }
-                }
-              ],
-              :restartPolicy => "Never"
-            }
-          }
-
-          spec[:spec][:imagePullPolicy]    = @pull_policy          if @pull_policy
-          spec[:spec][:serviceAccountName] = @task_service_account if @task_service_account
-
-          if secret
-            spec[:spec][:volumes] = [
-              {
-                :name   => "secret-volume",
-                :secret => {:secretName => secret}
-              }
-            ]
-
-            spec[:spec][:containers][0][:env] << {
-              :name  => "_CREDENTIALS",
-              :value => "/run/secrets/#{secret}/secret"
-            }
-
-            spec[:spec][:containers][0][:volumeMounts] = [
-              {
-                :name      => "secret-volume",
-                :mountPath => "/run/secrets/#{secret}",
-                :readOnly  => true
-              }
-            ]
-          end
-
-          spec
-        end
-
-        def create_pod!(name, image, env, secret = nil)
-          kubeclient.create_pod(pod_spec(name, image, env, secret))
-        end
-
-        def delete_pod!(name)
-          kubeclient.delete_pod(name, namespace)
-        end
-
-        def delete_pod(name)
-          delete_pod!(name)
-        rescue
-          nil
-        end
-
-        def create_secret!(secrets)
-          secret_name = SecureRandom.uuid
-
-          secret_config = {
-            :kind       => "Secret",
-            :apiVersion => "v1",
-            :metadata   => {
-              :name      => secret_name,
-              :namespace => namespace
-            },
-            :data       => {
-              :secret => Base64.urlsafe_encode64(secrets.to_json)
-            },
-            :type       => "Opaque"
-          }
-
-          kubeclient.create_secret(secret_config)
-
-          secret_name
-        end
-
-        def delete_secret!(secret_name)
-          kubeclient.delete_secret(secret_name, namespace)
-        end
-
-        def delete_secret(name)
-          delete_secret!(name)
-        rescue
-          nil
-        end
-
-        def kube_notice_type_to_event(type)
-          case type
-          when "ADDED"
-            :create
-          when "MODIFIED"
-            :update
-          when "DELETED"
-            :delete
-          else
-            :unknown
-          end
-        end
-
-        def error_notice?(notice)
-          return false unless notice.type == "ERROR"
-
-          message = notice.object&.message
-          code    = notice.object&.code
-          reason  = notice.object&.reason
-
-          logger.warn("Received [#{code} #{reason}], [#{message}]")
-
-          true
-        end
-
-        def parse_notice(notice)
-          return if notice.object.nil?
-
-          pod             = notice.object
-          container_ref   = pod.metadata.name
-          container_state = pod.to_h[:status].deep_stringify_keys
-
-          {"container_ref" => container_ref, "container_state" => container_state}
-        end
-
-        def kubeclient
-          return @kubeclient unless @kubeclient.nil?
-
-          if server && token
-            api_endpoint = server
-            auth_options = {:bearer_token => token}
-            ssl_options  = {:verify_ssl => verify_ssl}
-            ssl_options[:ca_file] = ca_file if ca_file
-          else
-            context = kubeconfig&.context(kubeconfig_context)
-            raise ArgumentError, "Missing connections options, provide a kubeconfig file or pass server and token via --docker-runner-options" if context.nil?
-
-            api_endpoint = context.api_endpoint
-            auth_options = context.auth_options
-            ssl_options  = context.ssl_options
-          end
-
-          @kubeclient = Kubeclient::Client.new(api_endpoint, "v1", :ssl_options => ssl_options, :auth_options => auth_options).tap(&:discover)
-        end
-
-        def kubeconfig
-          return if kubeconfig_file.nil? || !File.exist?(kubeconfig_file)
-
-          Kubeclient::Config.read(kubeconfig_file)
-        end
-      end
-    end
-  end
-end