floe 0.10.0 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b158e514e08902a1138c7b2878bb8633ca7c1abb59d4b77b6d9ce568c65710b
-  data.tar.gz: 32d053bba54e8c35645a636771964692e435ad72ee38e03c8c22b4da5bce25ec
+  metadata.gz: ddff07ec60a21f0d90224e992e9e08d51c420a66203daf3d2f9a46257b0df9fa
+  data.tar.gz: bd1dee4d43ea4e62bae056893342aec18fd3acfdcde4b916c1f8cb3f04f9607e
 SHA512:
-  metadata.gz: e39301eed1de9189b66f07a7ecdda807974bf47495b4906bd57f6f8ed862fab38448668155f79e12c87da8935f4054d14dbcd08e292429b194768d2234fa7c46
-  data.tar.gz: 5cf0476521a1cd4fc0dcc4edeccdd00c6c72ee88bf7428103b86f5f30097608f934f19e75499a61b5d7e7380f2e1451dccfd82f796eee6bcef041dd6f3db8664
+  metadata.gz: 54c7fc79d150ff6801ab36b686fea09543902ee8a76b2fb787a5e038bc2dfef6c55a33f628f4abb888c057db73ab51613aea545a643a64adbaffe6895bc1e8cb
+  data.tar.gz: 3b73fadfc840a7f69d923116005f174a07d7e8f9ddd878e852c979563bcd7b63ff1acc7461c9222c48bd598e169fe010a786f970c37029d1225c4fe78118a624

data/CHANGELOG.md

@@ -4,6 +4,30 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.11.1] - 2024-05-20
+### Fixed
+- Fix issue where a failed state can leave a workflow in "running" ([#182](https://github.com/ManageIQ/floe/pull/182))
+
+### Changed
+- Drop unused Task#status ([#180](https://github.com/ManageIQ/floe/pull/180))
+- Check task failed? in non_terminal_mixin ([#183](https://github.com/ManageIQ/floe/pull/183))
+
+## [0.11.0] - 2024-05-02
+### Fixed
+- Ensure the local code is loaded in exe/floe ([#173](https://github.com/ManageIQ/floe/pull/173))
+- Fix issues with exe/floe and various combinations of workflow and input ([#174](https://github.com/ManageIQ/floe/pull/174))
+
+### Added
+- Add support for pluggable schemes ([#169](https://github.com/ManageIQ/floe/pull/169))
+
+### Changed
+- Collapse some namespaces ([#171](https://github.com/ManageIQ/floe/pull/171))
+- Pass workflow context into runner#run_async! ([#177](https://github.com/ManageIQ/floe/pull/177))
+
+### Removed
+- Remove unused run! method ([#176](https://github.com/ManageIQ/floe/pull/176))
+
+
 ## [0.10.0] - 2024-04-05
 ### Fixed
 - Fix rubocops ([#164](https://github.com/ManageIQ/floe/pull/164))
@@ -149,7 +173,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.10.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.11.1...HEAD
+[0.11.1]: https://github.com/ManageIQ/floe/compare/v0.11.0...v0.11.1
+[0.11.0]: https://github.com/ManageIQ/floe/compare/v0.10.0...v0.11.0
 [0.10.0]: https://github.com/ManageIQ/floe/compare/v0.9.0...v0.10.0
 [0.9.0]: https://github.com/ManageIQ/floe/compare/v0.8.0...v0.9.0
 [0.8.0]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.8.0

data/exe/floe

@@ -1,45 +1,52 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "floe"
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
 require "optimist"
+require "floe"
+require "floe/container_runner"
 
 opts = Optimist.options do
   version("v#{Floe::VERSION}\n")
   usage("[options] workflow input [workflow2 input2]")
 
-  opt :workflow, "Path to your workflow json (legacy)",         :type => :string
-  opt :input, "JSON payload to input to the workflow (legacy)", :type => :string
-  opt :context, "JSON payload of the Context",                  :type => :string
-  opt :credentials, "JSON payload with credentials",            :type => :string
-  opt :credentials_file, "Path to a file with credentials",     :type => :string
-  opt :docker_runner, "Type of runner for docker images",       :type => :string, :short => 'r'
-  opt :docker_runner_options, "Options to pass to the runner",  :type => :strings, :short => 'o'
-
-  opt :docker,     "Use docker to run images     (short for --docker_runner=docker)",     :type => :boolean
-  opt :podman,     "Use podman to run images     (short for --docker_runner=podman)",     :type => :boolean
-  opt :kubernetes, "Use kubernetes to run images (short for --docker_runner=kubernetes)", :type => :boolean
+  opt :workflow, "Path to your workflow json file (alternative to passing a bare workflow)", :type => :string
+  opt :input, <<~EOMSG, :type => :string
+    JSON payload of the Input to the workflow
+      If --input is passed and --workflow is not passed, will be used for all bare workflows listed.
+      If --input is not passed and --workflow is passed, defaults to '{}'.
+  EOMSG
+  opt :context, "JSON payload of the Context",              :type => :string
+  opt :credentials, "JSON payload with Credentials",        :type => :string
+  opt :credentials_file, "Path to a file with Credentials", :type => :string
+
+  Floe::ContainerRunner.cli_options(self)
+
+  banner("")
+  banner("General options:")
 end
 
-# legacy support for --workflow
-args = ARGV.empty? ? [opts[:workflow], opts[:input]] : ARGV
-Optimist.die(:workflow, "must be specified") if args.empty?
+# Create workflow/input pairs from the various combinations of paramaters
+args =
+  if opts[:workflow_given]
+    Optimist.die("cannot specify both --workflow and bare workflows") if ARGV.any?
 
-# shortcut support
-opts[:docker_runner] ||= "docker" if opts[:docker]
-opts[:docker_runner] ||= "podman" if opts[:podman]
-opts[:docker_runner] ||= "kubernetes" if opts[:kubernetes]
+    [opts[:workflow], opts.fetch(:input, "{}")]
+  elsif opts[:input_given]
+    Optimist.die("workflow(s) must be specified") if ARGV.empty?
 
-require "logger"
-Floe.logger = Logger.new($stdout)
+    ARGV.flat_map { |w| [w, opts[:input].dup] }
+  else
+    Optimist.die("workflow/input pairs must be specified") if ARGV.empty? || (ARGV.size > 1 && ARGV.size.odd?)
 
-runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
+    ARGV
+  end
 
-begin
-  Floe.set_runner("docker", opts[:docker_runner], runner_options)
-rescue ArgumentError => e
-  Optimist.die(:docker_runner, e.message)
-end
+Floe::ContainerRunner.resolve_cli_options!(opts)
+
+require "logger"
+Floe.logger = Logger.new($stdout)
 
 credentials =
   if opts[:credentials_given]
@@ -50,7 +57,7 @@ credentials =
 
 workflows =
   args.each_slice(2).map do |workflow, input|
-    context = Floe::Workflow::Context.new(opts[:context], :input => input || opts[:input] || "{}")
+    context = Floe::Workflow::Context.new(opts[:context], :input => input)
     Floe::Workflow.load(workflow, context, credentials)
   end
 

data/lib/floe/container_runner/docker.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    class Docker < Floe::Runner
+      include Floe::ContainerRunner::DockerMixin
+
+      DOCKER_COMMAND = "docker"
+
+      def initialize(options = {})
+        require "awesome_spawn"
+        require "io/wait"
+        require "tempfile"
+
+        super
+
+        @network     = options.fetch("network", "bridge")
+        @pull_policy = options["pull-policy"]
+      end
+
+      def run_async!(resource, env = {}, secrets = {}, _context = {})
+        raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+        image = resource.sub("docker://", "")
+
+        runner_context = {}
+
+        if secrets && !secrets.empty?
+          runner_context["secrets_ref"] = create_secret(secrets)
+        end
+
+        begin
+          runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"])
+          runner_context
+        rescue AwesomeSpawn::CommandResultError => err
+          cleanup(runner_context)
+          {"Error" => "States.TaskFailed", "Cause" => err.to_s}
+        end
+      end
+
+      def cleanup(runner_context)
+        container_id, secrets_file = runner_context.values_at("container_ref", "secrets_ref")
+
+        delete_container(container_id) if container_id
+        delete_secret(secrets_file)    if secrets_file
+      end
+
+      def wait(timeout: nil, events: %i[create update delete], &block)
+        until_timestamp = Time.now.utc + timeout if timeout
+
+        r, w = IO.pipe
+
+        pid = AwesomeSpawn.run_detached(
+          self.class::DOCKER_COMMAND, :err => :out, :out => w, :params => wait_params(until_timestamp)
+        )
+
+        w.close
+
+        loop do
+          readable_timeout = until_timestamp - Time.now.utc if until_timestamp
+
+          # Wait for our end of the pipe to be readable and if it didn't timeout
+          # get the events from stdout
+          next if r.wait_readable(readable_timeout).nil?
+
+          # Get all events while the pipe is readable
+          notices = []
+          while r.ready?
+            notice = r.gets
+
+            # If the process has exited `r.gets` returns `nil` and the pipe is
+            # always `ready?`
+            break if notice.nil?
+
+            event, runner_context = parse_notice(notice)
+            next if event.nil? || !events.include?(event)
+
+            notices << [event, runner_context]
+          end
+
+          # If we're given a block yield the events otherwise return them
+          if block
+            notices.each(&block)
+          else
+            # Terminate the `docker events` process before returning the events
+            sigterm(pid)
+
+            return notices
+          end
+
+          # Check that the `docker events` process is still alive
+          Process.kill(0, pid)
+        rescue Errno::ESRCH
+          # Break out of the loop if the `docker events` process has exited
+          break
+        end
+      ensure
+        r.close
+      end
+
+      def status!(runner_context)
+        return if runner_context.key?("Error")
+
+        runner_context["container_state"] = inspect_container(runner_context["container_ref"])&.dig("State")
+      end
+
+      def running?(runner_context)
+        !!runner_context.dig("container_state", "Running")
+      end
+
+      def success?(runner_context)
+        runner_context.dig("container_state", "ExitCode") == 0
+      end
+
+      def output(runner_context)
+        return runner_context.slice("Error", "Cause") if runner_context.key?("Error")
+
+        output = docker!("logs", runner_context["container_ref"], :combined_output => true).output
+        runner_context["output"] = output
+      end
+
+      private
+
+      attr_reader :network
+
+      def run_container(image, env, secrets_file)
+        params = run_container_params(image, env, secrets_file)
+
+        logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
+
+        result = docker!(*params)
+        result.output
+      end
+
+      def run_container_params(image, env, secrets_file)
+        params  = ["run"]
+        params << :detach
+        params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+        params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
+        params << [:pull, @pull_policy] if @pull_policy
+        params << [:net, "host"] if @network == "host"
+        params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
+        params << [:name, container_name(image)]
+        params << image
+      end
+
+      def wait_params(until_timestamp)
+        params = ["events", [:format, "{{json .}}"], [:filter, "type=container"], [:since, Time.now.utc.to_i]]
+        params << [:until, until_timestamp.to_i] if until_timestamp
+        params
+      end
+
+      def parse_notice(notice)
+        notice = JSON.parse(notice)
+
+        status  = notice["status"]
+        event   = docker_event_status_to_event(status)
+        running = event != :delete
+
+        name, exit_code = notice.dig("Actor", "Attributes")&.values_at("name", "exitCode")
+
+        runner_context = {"container_ref" => name, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+        [event, runner_context]
+      rescue JSON::ParserError
+        []
+      end
+
+      def docker_event_status_to_event(status)
+        case status
+        when "create"
+          :create
+        when "start"
+          :update
+        when "die", "destroy"
+          :delete
+        else
+          :unkonwn
+        end
+      end
+
+      def inspect_container(container_id)
+        JSON.parse(docker!("inspect", container_id).output).first
+      rescue
+        nil
+      end
+
+      def delete_container(container_id)
+        docker!("rm", container_id)
+      rescue
+        nil
+      end
+
+      def delete_secret(secrets_file)
+        return unless File.exist?(secrets_file)
+
+        File.unlink(secrets_file)
+      rescue
+        nil
+      end
+
+      def create_secret(secrets)
+        secrets_file = Tempfile.new
+        secrets_file.write(secrets.to_json)
+        secrets_file.close
+        secrets_file.path
+      end
+
+      def sigterm(pid)
+        Process.kill("TERM", pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def global_docker_options
+        []
+      end
+
+      def docker!(*args, **kwargs)
+        params = global_docker_options + args
+        AwesomeSpawn.run!(self.class::DOCKER_COMMAND, :params => params, **kwargs)
+      end
+    end
+  end
+end

data/lib/floe/container_runner/docker_mixin.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    module DockerMixin
+      def image_name(image)
+        image.match(%r{^(?<repository>.+/)?(?<image>.+):(?<tag>.+)$})&.named_captures&.dig("image")
+      end
+
+      # 63 is the max kubernetes pod name length
+      # -5 for the "floe-" prefix
+      # -9 for the random hex suffix and leading hyphen
+      MAX_CONTAINER_NAME_SIZE = 63 - 5 - 9
+
+      def container_name(image)
+        name = image_name(image)
+        raise ArgumentError, "Invalid docker image [#{image}]" if name.nil?
+
+        # Normalize the image name to be used in the container name.
+        # This follows RFC 1123 Label names in Kubernetes as they are the most restrictive
+        # See https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+        # and https://github.com/kubernetes/kubernetes/blob/952a9cb0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L178-L184
+        #
+        # This does not follow the leading and trailing character restriction because we will embed it
+        # below with a prefix and suffix that already conform to the RFC.
+        normalized_name = name.downcase.gsub(/[^a-z0-9-]/, "-")[0, MAX_CONTAINER_NAME_SIZE]
+
+        "floe-#{normalized_name}-#{SecureRandom.hex(4)}"
+      end
+    end
+  end
+end