floe 0.7.1 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88cfa362fcb949aeaa6e9b0b799bff4bbb0e6b7158c95920545300ba063d6a0d
-  data.tar.gz: 14df3cddf7a1811e945f533b88fcacd6611566e212cacb88bebf2da0bae9fe67
+  metadata.gz: 8c7a74a5297258d481fb588ae0fa6eb1b22b7ecf5c049865b77ad23d6fb135cb
+  data.tar.gz: 82f73726e293e5345d3e7fa55a0049f881f2dce6e6d46570d2352968907c04b9
 SHA512:
-  metadata.gz: 90a3cdfdb241242d5b52ad403e7bbbf409476efec6d01d16f8f6f223b6f5fa3e227647da98ab2fb9466993982793be32684521ca5a268e73ab191ff7b6a025fb
-  data.tar.gz: c1194b741493a3db2f3abdf083ef2858b25af61442584c6a2001d222579c78e5057f0a7efd3c6c904e02d898c296f543c5af1889c2e4bb91c95f9197d327fa35
+  metadata.gz: 32d58e28cd76d936f31f9af2c1091d8a7dd930e47a2197b532c02d6d48df2c82feee696072af701aeca5f2af5437040ea3bace5df622c1ad5d0e47e388884ad2
+  data.tar.gz: 1ee0628fbfde496d00fae67812ac869582b1aca754aca7cf44caa7170edc1b0f6c9100a587770d6f6bb97bc0a948dd8fe0123fd108b9ed037ad71bc62bb8e104

data/CHANGELOG.md

@@ -4,8 +4,26 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
-## [0.7.1] - 2024-01-17
+## [0.9.0] - 2024-02-19
+### Changed
+- Default to wait indefinitely ([#157](https://github.com/ManageIQ/floe/pull/157))
+- Create docker runners factory and add scheme ([#152](https://github.com/ManageIQ/floe/pull/152))
+- Add a watch method to Workflow::Runner for event driven updates ([#95](https://github.com/ManageIQ/floe/pull/95))
+
+### Fixed
+- Fix waiting on extremely short durations ([#160](https://github.com/ManageIQ/floe/pull/160))
+- Fix wait state missing finish ([#159](https://github.com/ManageIQ/floe/pull/159))
+
+## [0.8.0] - 2024-01-17
+### Added
+- Add CLI shorthand options for docker runner ([#147](https://github.com/ManageIQ/floe/pull/147))
+- Run multiple workflows in exe/floe ([#149](https://github.com/ManageIQ/floe/pull/149))
+- Add secure options for passing credentials via command-line ([#151](https://github.com/ManageIQ/floe/pull/151))
+- Add a Docker Runner pull-policy option ([#155](https://github.com/ManageIQ/floe/pull/155))
+
 ### Fixed
+- Fix podman with empty output ([#150](https://github.com/ManageIQ/floe/pull/150))
+- Fix run_container logger saying docker when using podman ([#154](https://github.com/ManageIQ/floe/pull/154))
 - Ensure that workflow credentials is not-nil ([#156](https://github.com/ManageIQ/floe/pull/156))
 
 ## [0.7.0] - 2023-12-18
@@ -118,8 +136,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.7.1...HEAD
-[0.7.1]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.7.1
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.9.0...HEAD
+[0.9.0]: https://github.com/ManageIQ/floe/compare/v0.8.0...v0.9.0
+[0.8.0]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/ManageIQ/floe/compare/v0.6.1...v0.7.0
 [0.6.1]: https://github.com/ManageIQ/floe/compare/v0.6.0...v0.6.1
 [0.6.0]: https://github.com/ManageIQ/floe/compare/v0.5.0...v0.6.0

data/README.md

@@ -51,6 +51,16 @@ You can provide that at runtime via the `--credentials` parameter:
 bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"roleArn": "arn:aws:iam::111122223333:role/LambdaRole"}'
 ```
 
+Or if you are running the floe command programmatically you can securely provide the credentials via a stdin pipe via `--credentials=-`:
+```
+echo '{"roleArn": "arn:aws:iam::111122223333:role/LambdaRole"}' | bundle exec ruby exe/floe --workflow my-workflow.asl --credentials -
+```
+
+Or you can pass a file path with the `--credentials-file` parameter:
+```
+bundle exec ruby exe/floe --workflow my-workflow.asl --credentials-file /tmp/20231218-80537-kj494t
+```
+
 If you need to set a credential at runtime you can do that by using the `"ResultPath": "$.Credentials"` directive, for example to user a username/password to login and get a Bearer token:
 
 ```
@@ -152,6 +162,7 @@ end
 Options supported by the Docker docker runner are:
 
 * `network` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+* `pull-policy` - Pull image policy. The default is missing.  Allowed values: always, missing, never
 
 #### Podman
 
@@ -161,6 +172,7 @@ Options supported by the podman docker runner are:
 * `log-level=string` - Log messages above specified level (trace, debug, info, warn, warning, error, fatal, panic)
 * `network=string` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
 * `noout=boolean` - do not output to stdout
+* `pull-policy=string` - Pull image policy. The default is missing. Allowed values: always, missing, never, newer
 * `root=string` - Path to the root directory in which data, including images, is stored
 * `runroot=string` - Path to the 'run directory' where all state information is stored
 * `runtime=string` - Path to the OCI-compatible binary used to run containers
@@ -179,6 +191,7 @@ Options supported by the kubernetes docker runner are:
 * `kubeconfig` - Path to a kubeconfig file, defaults to `KUBECONFIG` environment variable or `~/.kube/config`
 * `kubeconfig_context` - Context to use in the kubeconfig file, defaults to `"default"`
 * `namespace` - Namespace to use when creating kubernetes resources, defaults to `"default"`
+* `pull-policy` - Pull image policy. The default is Always.  Allowed values: IfNotPresent, Always, Never
 * `server` - A kubernetes API Server URL, overrides anything in your kubeconfig file.  If set `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` will be used
 * `token` - A bearer_token to use to authenticate to the kubernetes API, overrides anything in your kubeconfig file.  If present, `/run/secrets/kubernetes.io/serviceaccount/token` will be used
 * `ca_file` - Path to a certificate-authority file for the kubernetes API, only valid if server and token are passed.  If present `/run/secrets/kubernetes.io/serviceaccount/ca.crt` will be used

data/exe/floe

@@ -6,35 +6,64 @@ require "optimist"
 
 opts = Optimist.options do
   version("v#{Floe::VERSION}\n")
-  opt :workflow, "Path to your workflow json",                 :type => :string, :required => true
-  opt :input, "JSON payload to input to the workflow",         :default => '{}'
-  opt :credentials, "JSON payload with credentials",           :default => "{}"
-  opt :docker_runner, "Type of runner for docker images",      :default => "docker"
-  opt :docker_runner_options, "Options to pass to the runner", :type => :strings
+  usage("[options] workflow input [workflow2 input2]")
+
+  opt :workflow, "Path to your workflow json (legacy)",         :type => :string
+  opt :input, "JSON payload to input to the workflow (legacy)", :type => :string
+  opt :credentials, "JSON payload with credentials",            :type => :string
+  opt :credentials_file, "Path to a file with credentials",     :type => :string
+  opt :docker_runner, "Type of runner for docker images",       :type => :string, :short => 'r'
+  opt :docker_runner_options, "Options to pass to the runner",  :type => :strings, :short => 'o'
+
+  opt :docker,     "Use docker to run images     (short for --docker_runner=docker)",     :type => :boolean
+  opt :podman,     "Use podman to run images     (short for --docker_runner=podman)",     :type => :boolean
+  opt :kubernetes, "Use kubernetes to run images (short for --docker_runner=kubernetes)", :type => :boolean
 end
 
-Optimist.die(:docker_runner, "must be one of #{Floe::Workflow::Runner::TYPES.join(", ")}") unless Floe::Workflow::Runner::TYPES.include?(opts[:docker_runner])
+# legacy support for --workflow
+args = ARGV.empty? ? [opts[:workflow], opts[:input]] : ARGV
+Optimist.die(:workflow, "must be specified") if args.empty?
+
+# shortcut support
+opts[:docker_runner] ||= "docker" if opts[:docker]
+opts[:docker_runner] ||= "podman" if opts[:podman]
+opts[:docker_runner] ||= "kubernetes" if opts[:kubernetes]
 
 require "logger"
 Floe.logger = Logger.new($stdout)
 
-runner_klass = case opts[:docker_runner]
-               when "docker"
-                 Floe::Workflow::Runner::Docker
-               when "podman"
-                 Floe::Workflow::Runner::Podman
-               when "kubernetes"
-                 Floe::Workflow::Runner::Kubernetes
-               end
-
 runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
 
-Floe::Workflow::Runner.docker_runner = runner_klass.new(runner_options)
+begin
+  Floe.set_runner("docker", opts[:docker_runner], runner_options)
+rescue ArgumentError => e
+  Optimist.die(:docker_runner, e.message)
+end
+
+credentials =
+  if opts[:credentials_given]
+    opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
+  elsif opts[:credentials_file_given]
+    File.read(opts[:credentials_file])
+  end
 
-context = Floe::Workflow::Context.new(:input => opts[:input])
-workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
+workflows =
+  args.each_slice(2).map do |workflow, input|
+    context = Floe::Workflow::Context.new(:input => input || opts[:input] || "{}")
+    Floe::Workflow.load(workflow, context, credentials)
+  end
+
+# run
+
+Floe::Workflow.wait(workflows, &:run_nonblock)
+
+# display status
+
+workflows.each do |workflow|
+  puts "", "#{workflow.name}#{" (#{workflow.status})" unless workflow.context.success?}", "===" if workflows.size > 1
+  puts workflow.output.inspect
+end
 
-workflow.run!
+# exit status
 
-puts workflow.output.inspect
-exit workflow.status == "success" ? 0 : 1
+exit workflows.all? { |workflow| workflow.context.success? } ? 0 : 1

data/floe.gemspec

@@ -29,7 +29,8 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "awesome_spawn", "~>1.0"
+  spec.add_dependency "awesome_spawn", "~>1.6"
+  spec.add_dependency "io-wait"
   spec.add_dependency "jsonpath", "~>1.1"
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "optimist", "~>3.0"

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.7.1".freeze
+  VERSION = "0.9.0".freeze
 end

data/lib/floe/workflow/context.rb

@@ -74,6 +74,10 @@ module Floe
         end
       end
 
+      def success?
+        status == "success"
+      end
+
       def state=(val)
         @context["State"] = val
       end

data/lib/floe/workflow/runner/docker.rb

@@ -10,11 +10,13 @@ module Floe
 
         def initialize(options = {})
           require "awesome_spawn"
+          require "io/wait"
           require "tempfile"
 
           super
 
-          @network = options.fetch("network", "bridge")
+          @network     = options.fetch("network", "bridge")
+          @pull_policy = options["pull-policy"]
         end
 
         def run_async!(resource, env = {}, secrets = {})
@@ -44,10 +46,63 @@ module Floe
           delete_secret(secrets_file)    if secrets_file
         end
 
+        def wait(timeout: nil, events: %i[create update delete], &block)
+          until_timestamp = Time.now.utc + timeout if timeout
+
+          r, w = IO.pipe
+
+          pid = AwesomeSpawn.run_detached(
+            self.class::DOCKER_COMMAND, :err => :out, :out => w, :params => wait_params(until_timestamp)
+          )
+
+          w.close
+
+          loop do
+            readable_timeout = until_timestamp - Time.now.utc if until_timestamp
+
+            # Wait for our end of the pipe to be readable and if it didn't timeout
+            # get the events from stdout
+            next if r.wait_readable(readable_timeout).nil?
+
+            # Get all events while the pipe is readable
+            notices = []
+            while r.ready?
+              notice = r.gets
+
+              # If the process has exited `r.gets` returns `nil` and the pipe is
+              # always `ready?`
+              break if notice.nil?
+
+              event, runner_context = parse_notice(notice)
+              next if event.nil? || !events.include?(event)
+
+              notices << [event, runner_context]
+            end
+
+            # If we're given a block yield the events otherwise return them
+            if block
+              notices.each(&block)
+            else
+              # Terminate the `docker events` process before returning the events
+              sigterm(pid)
+
+              return notices
+            end
+
+            # Check that the `docker events` process is still alive
+            Process.kill(0, pid)
+          rescue Errno::ESRCH
+            # Break out of the loop if the `docker events` process has exited
+            break
+          end
+        ensure
+          r.close
+        end
+
         def status!(runner_context)
           return if runner_context.key?("Error")
 
-          runner_context["container_state"] = inspect_container(runner_context["container_ref"]).first&.dig("State")
+          runner_context["container_state"] = inspect_container(runner_context["container_ref"])&.dig("State")
         end
 
         def running?(runner_context)
@@ -72,7 +127,7 @@ module Floe
         def run_container(image, env, secrets_file)
           params = run_container_params(image, env, secrets_file)
 
-          logger.debug("Running #{AwesomeSpawn.build_command_line("docker", params)}")
+          logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
 
           result = docker!(*params)
           result.output
@@ -83,14 +138,52 @@ module Floe
           params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
           params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
+          params << [:pull, @pull_policy] if @pull_policy
           params << [:net, "host"] if @network == "host"
           params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
           params << [:name, container_name(image)]
           params << image
         end
 
+        def wait_params(until_timestamp)
+          params = ["events", [:format, "{{json .}}"], [:filter, "type=container"], [:since, Time.now.utc.to_i]]
+          params << [:until, until_timestamp.to_i] if until_timestamp
+          params
+        end
+
+        def parse_notice(notice)
+          notice = JSON.parse(notice)
+
+          status  = notice["status"]
+          event   = docker_event_status_to_event(status)
+          running = event != :delete
+
+          name, exit_code = notice.dig("Actor", "Attributes")&.values_at("name", "exitCode")
+
+          runner_context = {"container_ref" => name, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+          [event, runner_context]
+        rescue JSON::ParserError
+          []
+        end
+
+        def docker_event_status_to_event(status)
+          case status
+          when "create"
+            :create
+          when "start"
+            :update
+          when "die", "destroy"
+            :delete
+          else
+            :unkonwn
+          end
+        end
+
         def inspect_container(container_id)
-          JSON.parse(docker!("inspect", container_id).output)
+          JSON.parse(docker!("inspect", container_id).output).first
+        rescue
+          nil
         end
 
         def delete_container(container_id)
@@ -114,6 +207,12 @@ module Floe
           secrets_file.path
         end
 
+        def sigterm(pid)
+          Process.kill("TERM", pid)
+        rescue Errno::ESRCH
+          nil
+        end
+
         def global_docker_options
           []
         end

data/lib/floe/workflow/runner/kubernetes.rb

@@ -40,6 +40,7 @@ module Floe
 
           @namespace = options.fetch("namespace", "default")
 
+          @pull_policy          = options["pull-policy"]
           @task_service_account = options["task_service_account"]
 
           super
@@ -52,7 +53,7 @@ module Floe
           name   = container_name(image)
           secret = create_secret!(secrets) if secrets && !secrets.empty?
 
-          runner_context = {"container_ref" => name, "secrets_ref" => secret}
+          runner_context = {"container_ref" => name, "container_state" => {"phase" => "Pending"}, "secrets_ref" => secret}
 
           begin
             create_pod!(name, image, env, secret)
@@ -101,6 +102,54 @@ module Floe
           delete_secret(secret) if secret
         end
 
+        def wait(timeout: nil, events: %i[create update delete])
+          retry_connection = true
+
+          begin
+            watcher = kubeclient.watch_pods(:namespace => namespace)
+
+            retry_connection = true
+
+            if timeout.to_i > 0
+              timeout_thread = Thread.new do
+                sleep(timeout)
+                watcher.finish
+              end
+            end
+
+            watcher.each do |notice|
+              break if error_notice?(notice)
+
+              event = kube_notice_type_to_event(notice.type)
+              next unless events.include?(event)
+
+              runner_context = parse_notice(notice)
+              next if runner_context.nil?
+
+              if block_given?
+                yield [event, runner_context]
+              else
+                timeout_thread&.kill # If we break out before the timeout, kill the timeout thread
+                return [[event, runner_context]]
+              end
+            end
+          rescue Kubeclient::HttpError => err
+            raise unless err.error_code == 401 && retry_connection
+
+            @kubeclient = nil
+            retry_connection = false
+            retry
+          ensure
+            begin
+              watch&.finish
+            rescue
+              nil
+            end
+
+            timeout_thread&.join(0)
+          end
+        end
+
         private
 
         attr_reader :ca_file, :kubeconfig_file, :kubeconfig_context, :namespace, :server, :token, :verify_ssl
@@ -143,6 +192,7 @@ module Floe
             }
           }
 
+          spec[:spec][:imagePullPolicy]    = @pull_policy          if @pull_policy
           spec[:spec][:serviceAccountName] = @task_service_account if @task_service_account
 
           if secret
@@ -215,6 +265,41 @@ module Floe
           nil
         end
 
+        def kube_notice_type_to_event(type)
+          case type
+          when "ADDED"
+            :create
+          when "MODIFIED"
+            :update
+          when "DELETED"
+            :delete
+          else
+            :unknown
+          end
+        end
+
+        def error_notice?(notice)
+          return false unless notice.type == "ERROR"
+
+          message = notice.object&.message
+          code    = notice.object&.code
+          reason  = notice.object&.reason
+
+          logger.warn("Received [#{code} #{reason}], [#{message}]")
+
+          true
+        end
+
+        def parse_notice(notice)
+          return if notice.object.nil?
+
+          pod             = notice.object
+          container_ref   = pod.metadata.name
+          container_state = pod.to_h[:status].deep_stringify_keys
+
+          {"container_ref" => container_ref, "container_state" => container_state}
+        end
+
         def kubeclient
           return @kubeclient unless @kubeclient.nil?
 

data/lib/floe/workflow/runner/podman.rb

@@ -16,6 +16,7 @@ module Floe
           @log_level       = options["log-level"]
           @network         = options["network"]
           @noout           = options["noout"].to_s == "true" if options.key?("noout")
+          @pull_policy     = options["pull-policy"]
           @root            = options["root"]
           @runroot         = options["runroot"]
           @runtime         = options["runtime"]
@@ -35,7 +36,8 @@ module Floe
           params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
           params << [:e, "_CREDENTIALS=/run/secrets/#{secret}"] if secret
-          params << [:net, "host"] if @network == "host"
+          params << [:pull, @pull_policy] if @pull_policy
+          params << [:net, "host"]        if @network == "host"
           params << [:secret, secret] if secret
           params << [:name, container_name(image)]
           params << image
@@ -53,6 +55,32 @@ module Floe
           nil
         end
 
+        def parse_notice(notice)
+          id, status, exit_code = JSON.parse(notice).values_at("ID", "Status", "ContainerExitCode")
+
+          event   = podman_event_status_to_event(status)
+          running = event != :delete
+
+          runner_context = {"container_ref" => id, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+          [event, runner_context]
+        rescue JSON::ParserError
+          []
+        end
+
+        def podman_event_status_to_event(status)
+          case status
+          when "create"
+            :create
+          when "init", "start"
+            :update
+          when "died", "cleanup", "remove"
+            :delete
+          else
+            :unknown
+          end
+        end
+
         alias podman! docker!
 
         def global_docker_options

data/lib/floe/workflow/runner.rb

@@ -5,29 +5,42 @@ module Floe
     class Runner
       include Logging
 
-      TYPES         = %w[docker podman kubernetes].freeze
       OUTPUT_MARKER = "__FLOE_OUTPUT__\n"
 
       def initialize(_options = {})
       end
 
+      @runners = {}
       class << self
-        attr_writer :docker_runner
+        # deprecated -- use Floe.set_runner instead
+        def docker_runner=(value)
+          set_runner("docker", value)
+        end
 
-        def docker_runner
-          @docker_runner ||= Floe::Workflow::Runner::Docker.new
+        # see Floe.set_runner
+        def set_runner(scheme, name_or_instance, options = {})
+          @runners[scheme] =
+            case name_or_instance
+            when "docker", nil
+              Floe::Workflow::Runner::Docker.new(options)
+            when "podman"
+              Floe::Workflow::Runner::Podman.new(options)
+            when "kubernetes"
+              Floe::Workflow::Runner::Kubernetes.new(options)
+            when Floe::Workflow::Runner
+              name_or_instance
+            else
+              raise ArgumentError, "docker runner must be one of: docker, podman, kubernetes"
+            end
         end
 
         def for_resource(resource)
           raise ArgumentError, "resource cannot be nil" if resource.nil?
 
+          # if no runners are set, default docker:// to docker
+          set_runner("docker", "docker") if @runners.empty?
           scheme = resource.split("://").first
-          case scheme
-          when "docker"
-            docker_runner
-          else
-            raise "Invalid resource scheme [#{scheme}]"
-          end
+          @runners[scheme] || raise(ArgumentError, "Invalid resource scheme [#{scheme}]")
         end
       end
 
@@ -55,6 +68,10 @@ module Floe
       def cleanup(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
+
+      def wait(timeout: nil, events: %i[create update delete])
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
     end
   end
 end

data/lib/floe/workflow/state.rb

@@ -33,16 +33,12 @@ module Floe
         raise Floe::InvalidWorkflowError, "State name [#{name}] must be less than or equal to 80 characters" if name.length > 80
       end
 
-      def run!(_input = nil)
-        wait until run_nonblock! == 0
-      end
-
-      def wait(timeout: 5)
+      def wait(timeout: nil)
         start = Time.now.utc
 
         loop do
           return 0             if ready?
-          return Errno::EAGAIN if timeout.zero? || Time.now.utc - start > timeout
+          return Errno::EAGAIN if timeout && (timeout.zero? || Time.now.utc - start > timeout)
 
           sleep(1)
         end
@@ -97,6 +93,14 @@ module Floe
         context.state.key?("FinishedTime")
       end
 
+      def waiting?
+        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
+      end
+
+      def wait_until
+        context.state["WaitUntil"] && Time.parse(context.state["WaitUntil"])
+      end
+
       private
 
       def wait_until!(seconds: nil, time: nil)
@@ -109,10 +113,6 @@ module Floe
             time.iso8601
           end
       end
-
-      def waiting?
-        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
-      end
     end
   end
 end

data/lib/floe/workflow/states/non_terminal_mixin.rb

@@ -4,6 +4,11 @@ module Floe
   class Workflow
     module States
       module NonTerminalMixin
+        def finish
+          context.next_state = end? ? nil : @next
+          super
+        end
+
         def validate_state_next!
           raise Floe::InvalidWorkflowError, "Missing \"Next\" field in state [#{name}]" if @next.nil? && !@end
           raise Floe::InvalidWorkflowError, "\"Next\" [#{@next}] not in \"States\" for state [#{name}]" if @next && !workflow.payload["States"].key?(@next)

data/lib/floe/workflow/states/task.rb

@@ -46,18 +46,19 @@ module Floe
         end
 
         def finish
+          super
+
           output = runner.output(context.state["RunnerContext"])
 
           if success?
             output = parse_output(output)
             context.state["Output"] = process_output(context.input.dup, output)
-            context.next_state      = next_state
           else
+            context.next_state = nil
             error = parse_error(output)
             retry_state!(error) || catch_error!(error) || fail_workflow!(error)
           end
 
-          super
         ensure
           runner.cleanup(context.state["RunnerContext"])
         end
@@ -137,8 +138,8 @@ module Floe
         end
 
         def parse_output(output)
-          return if output.nil?
           return output if output.kind_of?(Hash)
+          return if output.nil? || output.empty?
 
           JSON.parse(output.split("\n").last)
         rescue JSON::ParserError

data/lib/floe/workflow/states/wait.rb

@@ -28,10 +28,9 @@ module Floe
 
         def start(input)
           super
-          input = input_path.value(context, input)
 
-          context.output     = output_path.value(context, input)
-          context.next_state = end? ? nil : @next
+          input          = input_path.value(context, input)
+          context.output = output_path.value(context, input)
 
           wait_until!(
             :seconds => seconds_path ? seconds_path.value(context, input).to_i : seconds,

data/lib/floe/workflow.rb

@@ -8,32 +8,86 @@ module Floe
     include Logging
 
     class << self
-      def load(path_or_io, context = nil, credentials = {})
+      def load(path_or_io, context = nil, credentials = {}, name = nil)
         payload = path_or_io.respond_to?(:read) ? path_or_io.read : File.read(path_or_io)
-        new(payload, context, credentials)
+        # default the name if it is a filename and none was passed in
+        name ||= path_or_io.respond_to?(:read) ? "stream" : path_or_io.split("/").last.split(".").first
+
+        new(payload, context, credentials, name)
       end
 
-      def wait(workflows, timeout: 5)
+      def wait(workflows, timeout: nil, &block)
+        workflows = [workflows] if workflows.kind_of?(self)
         logger.info("checking #{workflows.count} workflows...")
 
-        start = Time.now.utc
-        ready = []
+        run_until   = Time.now.utc + timeout if timeout.to_i > 0
+        ready       = []
+        queue       = Queue.new
+        wait_thread = Thread.new do
+          loop do
+            Runner.for_resource("docker").wait do |event, runner_context|
+              queue.push([event, runner_context])
+            end
+          end
+        end
 
         loop do
           ready = workflows.select(&:step_nonblock_ready?)
-          break if timeout.zero? || Time.now.utc - start > timeout || !ready.empty?
-
-          sleep(1)
+          break if block.nil? && !ready.empty?
+
+          ready.each(&block)
+
+          # Break if all workflows are completed or we've exceeded the
+          # requested timeout
+          break if workflows.all?(&:end?)
+          break if timeout && (timeout.zero? || Time.now.utc > run_until)
+
+          # Find the earliest time that we should wakeup if no container events
+          # are caught, either a workflow in a Wait or Retry state or we've
+          # exceeded the requested timeout
+          wait_until = workflows.map(&:wait_until)
+                                .unshift(run_until)
+                                .compact
+                                .min
+
+          # If a workflow is in a waiting state wakeup the main thread when
+          # it will be done sleeping
+          if wait_until
+            sleep_thread = Thread.new do
+              sleep_duration = wait_until - Time.now.utc
+              sleep sleep_duration if sleep_duration > 0
+              queue.push(nil)
+            end
+          end
+
+          loop do
+            # Block until an event is raised
+            event, runner_context = queue.pop
+            break if event.nil?
+
+            # If the event is for one of our workflows set the updated runner_context
+            workflows.each do |workflow|
+              next unless workflow.context.state.dig("RunnerContext", "container_ref") == runner_context["container_ref"]
+
+              workflow.context.state["RunnerContext"] = runner_context
+            end
+
+            break if queue.empty?
+          end
+        ensure
+          sleep_thread&.kill
         end
 
         logger.info("checking #{workflows.count} workflows...Complete - #{ready.count} ready")
         ready
+      ensure
+        wait_thread&.kill
       end
     end
 
-    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at
+    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at, :name
 
-    def initialize(payload, context = nil, credentials = {})
+    def initialize(payload, context = nil, credentials = {}, name = nil)
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
       credentials = JSON.parse(credentials) if credentials.kind_of?(String)
       context     = Context.new(context)    unless context.kind_of?(Context)
@@ -42,12 +96,13 @@ module Floe
       raise Floe::InvalidWorkflowError, "Missing field \"StartAt\"" if payload["StartAt"].nil?
       raise Floe::InvalidWorkflowError, "\"StartAt\" not in the \"States\" field" unless payload["States"].key?(payload["StartAt"])
 
+      @name        = name
       @payload     = payload
       @context     = context
       @credentials = credentials || {}
       @start_at    = payload["StartAt"]
 
-      @states         = payload["States"].to_a.map { |name, state| State.build!(self, name, state) }
+      @states         = payload["States"].to_a.map { |state_name, state| State.build!(self, state_name, state) }
       @states_by_name = @states.each_with_object({}) { |state, result| result[state.name] = state }
 
       unless context.state.key?("Name")
@@ -58,16 +113,6 @@ module Floe
       raise Floe::InvalidWorkflowError, err.message
     end
 
-    def run!
-      step until end?
-      self
-    end
-
-    def step
-      step_nonblock_wait until step_nonblock == 0
-      self
-    end
-
     def run_nonblock
       loop while step_nonblock == 0 && !end?
       self
@@ -80,7 +125,7 @@ module Floe
       current_state.run_nonblock!
     end
 
-    def step_nonblock_wait(timeout: 5)
+    def step_nonblock_wait(timeout: nil)
       current_state.wait(:timeout => timeout)
     end
 
@@ -88,6 +133,14 @@ module Floe
       current_state.ready?
     end
 
+    def waiting?
+      current_state.waiting?
+    end
+
+    def wait_until
+      current_state.wait_until
+    end
+
     def status
       context.status
     end

data/lib/floe.rb

@@ -45,7 +45,27 @@ module Floe
     @logger ||= NullLogger.new
   end
 
+  # Set the logger to use
+  #
+  # @example
+  #   require "logger"
+  #   Floe.logger = Logger.new($stdout)
+  #
+  # @param logger [Logger] logger to use for logging actions
   def self.logger=(logger)
     @logger = logger
   end
+
+  # Set the runner to use
+  #
+  # @example
+  #   Floe.set_runner "docker", kubernetes", {}
+  #   Floe.set_runner "docker", Floe::Workflow::Runner::Kubernetes.new({})
+  #
+  # @param scheme [String] scheme                           Protocol to register (e.g.: docker)
+  # @param name_or_instance [String|Floe::Workflow::Runner] Name of runner to use for docker (e.g.: docker)
+  # @param options [Hash]                                   Options for constructor of the runner (optional)
+  def self.set_runner(scheme, name_or_instance, options = {})
+    Floe::Workflow::Runner.set_runner(scheme, name_or_instance, options)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.9.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-02-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: io-wait
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement