floe 0.8.0 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c121968c8c3f9c70edaa20c0ab3e7937206623b7c06c0cf0f300aceddcde7814
-  data.tar.gz: 943d9e77a9e8309a771e57fee5d6f70e5026a4d254b0da5072b768a99f427d1a
+  metadata.gz: 2b158e514e08902a1138c7b2878bb8633ca7c1abb59d4b77b6d9ce568c65710b
+  data.tar.gz: 32d053bba54e8c35645a636771964692e435ad72ee38e03c8c22b4da5bce25ec
 SHA512:
-  metadata.gz: 4fa820c364e8dc3266c2fc1eb29d682e72a9d2ab4d8f154aac3a88ff6d4c35e45e124fe714b80110b52e78a0ab654e25f956bc8a39f45f05f3990c0ab26d01ed
-  data.tar.gz: b62ac48c4105497c1d3dd2f2ef9d8aec60c25bcadf71046e8488175705365d0d84facde34c11f0047660b5ab2f4390f13babbe8ba2ab6becf31027f027e0b833
+  metadata.gz: e39301eed1de9189b66f07a7ecdda807974bf47495b4906bd57f6f8ed862fab38448668155f79e12c87da8935f4054d14dbcd08e292429b194768d2234fa7c46
+  data.tar.gz: 5cf0476521a1cd4fc0dcc4edeccdd00c6c72ee88bf7428103b86f5f30097608f934f19e75499a61b5d7e7380f2e1451dccfd82f796eee6bcef041dd6f3db8664

data/CHANGELOG.md

@@ -4,6 +4,29 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.10.0] - 2024-04-05
+### Fixed
+- Fix rubocops ([#164](https://github.com/ManageIQ/floe/pull/164))
+- Output should contain errors ([#165](https://github.com/ManageIQ/floe/pull/165))
+
+### Added
+- Add simplecov ([#162](https://github.com/ManageIQ/floe/pull/162))
+- Add ability to pass context on the command line ([#161](https://github.com/ManageIQ/floe/pull/161))
+- Add specs for `Workflow#wait_until`, `#waiting?` ([#166](https://github.com/ManageIQ/floe/pull/166))
+
+### Changed
+- Drop non-standard Error/Cause fields ([#167](https://github.com/ManageIQ/floe/pull/167))
+
+## [0.9.0] - 2024-02-19
+### Changed
+- Default to wait indefinitely ([#157](https://github.com/ManageIQ/floe/pull/157))
+- Create docker runners factory and add scheme ([#152](https://github.com/ManageIQ/floe/pull/152))
+- Add a watch method to Workflow::Runner for event driven updates ([#95](https://github.com/ManageIQ/floe/pull/95))
+
+### Fixed
+- Fix waiting on extremely short durations ([#160](https://github.com/ManageIQ/floe/pull/160))
+- Fix wait state missing finish ([#159](https://github.com/ManageIQ/floe/pull/159))
+
 ## [0.8.0] - 2024-01-17
 ### Added
 - Add CLI shorthand options for docker runner ([#147](https://github.com/ManageIQ/floe/pull/147))
@@ -126,7 +149,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.8.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.10.0...HEAD
+[0.10.0]: https://github.com/ManageIQ/floe/compare/v0.9.0...v0.10.0
+[0.9.0]: https://github.com/ManageIQ/floe/compare/v0.8.0...v0.9.0
 [0.8.0]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/ManageIQ/floe/compare/v0.6.1...v0.7.0
 [0.6.1]: https://github.com/ManageIQ/floe/compare/v0.6.0...v0.6.1

data/Gemfile

@@ -7,9 +7,3 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
 
 # Specify your gem's dependencies in floe.gemspec
 gemspec
-
-gem "manageiq-style"
-gem "rake", "~> 13.0"
-gem "rspec"
-gem "rubocop"
-gem "timecop"

data/exe/floe

@@ -10,6 +10,7 @@ opts = Optimist.options do
 
   opt :workflow, "Path to your workflow json (legacy)",         :type => :string
   opt :input, "JSON payload to input to the workflow (legacy)", :type => :string
+  opt :context, "JSON payload of the Context",                  :type => :string
   opt :credentials, "JSON payload with credentials",            :type => :string
   opt :credentials_file, "Path to a file with credentials",     :type => :string
   opt :docker_runner, "Type of runner for docker images",       :type => :string, :short => 'r'
@@ -20,8 +21,6 @@ opts = Optimist.options do
   opt :kubernetes, "Use kubernetes to run images (short for --docker_runner=kubernetes)", :type => :boolean
 end
 
-Optimist.die(:docker_runner, "must be one of #{Floe::Workflow::Runner::TYPES.join(", ")}") unless Floe::Workflow::Runner::TYPES.include?(opts[:docker_runner])
-
 # legacy support for --workflow
 args = ARGV.empty? ? [opts[:workflow], opts[:input]] : ARGV
 Optimist.die(:workflow, "must be specified") if args.empty?
@@ -34,18 +33,13 @@ opts[:docker_runner] ||= "kubernetes" if opts[:kubernetes]
 require "logger"
 Floe.logger = Logger.new($stdout)
 
-runner_klass = case opts[:docker_runner]
-               when "docker"
-                 Floe::Workflow::Runner::Docker
-               when "podman"
-                 Floe::Workflow::Runner::Podman
-               when "kubernetes"
-                 Floe::Workflow::Runner::Kubernetes
-               end
-
 runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
 
-Floe::Workflow::Runner.docker_runner = runner_klass.new(runner_options)
+begin
+  Floe.set_runner("docker", opts[:docker_runner], runner_options)
+rescue ArgumentError => e
+  Optimist.die(:docker_runner, e.message)
+end
 
 credentials =
   if opts[:credentials_given]
@@ -56,19 +50,13 @@ credentials =
 
 workflows =
   args.each_slice(2).map do |workflow, input|
-    context = Floe::Workflow::Context.new(:input => input || opts[:input] || "{}")
+    context = Floe::Workflow::Context.new(opts[:context], :input => input || opts[:input] || "{}")
     Floe::Workflow.load(workflow, context, credentials)
   end
 
 # run
 
-outstanding = workflows.dup
-until outstanding.empty?
-  ready = outstanding.select(&:step_nonblock_ready?)
-  ready.map(&:run_nonblock)
-  outstanding -= ready.select(&:end?)
-  sleep(1) if !outstanding.empty?
-end
+Floe::Workflow.wait(workflows, &:run_nonblock)
 
 # display status
 

data/floe.gemspec

@@ -29,8 +29,16 @@ Gem::Specification.new do |spec|
   spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "awesome_spawn", "~>1.0"
+  spec.add_dependency "awesome_spawn", "~>1.6"
+  spec.add_dependency "io-wait"
   spec.add_dependency "jsonpath", "~>1.1"
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "optimist", "~>3.0"
+
+  spec.add_development_dependency "manageiq-style"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov", ">= 0.21.2"
+  spec.add_development_dependency "timecop"
 end

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.8.0".freeze
+  VERSION = "0.10.0"
 end

data/lib/floe/workflow/context.rb

@@ -3,19 +3,19 @@
 module Floe
   class Workflow
     class Context
+      # @param context [Json|Hash] (default, create another with input and execution params)
+      # @param input [Hash] (default: {})
       def initialize(context = nil, input: {})
         context = JSON.parse(context) if context.kind_of?(String)
         input = JSON.parse(input) if input.kind_of?(String)
 
-        @context = context || {
-          "Execution"    => {
-            "Input" => input
-          },
-          "State"        => {},
-          "StateHistory" => [],
-          "StateMachine" => {},
-          "Task"         => {}
-        }
+        @context = context || {}
+        self["Execution"]          ||= {}
+        self["Execution"]["Input"] ||= input
+        self["State"]              ||= {}
+        self["StateHistory"]       ||= []
+        self["StateMachine"]       ||= {}
+        self["Task"]               ||= {}
       end
 
       def execution
@@ -30,6 +30,10 @@ module Floe
         started? && !ended?
       end
 
+      def failed?
+        output&.key?("Error") || false
+      end
+
       def ended?
         execution.key?("EndTime")
       end
@@ -67,7 +71,7 @@ module Floe
           "pending"
         elsif running?
           "running"
-        elsif state["Error"]
+        elsif failed?
           "failure"
         else
           "success"

data/lib/floe/workflow/reference_path.rb

@@ -19,11 +19,11 @@ module Floe
         super
 
         raise Floe::InvalidWorkflowError, "Invalid Reference Path" if payload.match?(/@|,|:|\?/)
+
         @path = JsonPath.new(payload)
                         .path[1..]
                         .map { |v| v.match(/\[(?<name>.+)\]/)["name"] }
-                        .map { |v| v[0] == "'" ? v.delete("'") : v.to_i }
-                        .compact
+                        .filter_map { |v| v[0] == "'" ? v.delete("'") : v.to_i }
       end
 
       def get(context)

data/lib/floe/workflow/runner/docker.rb

@@ -10,6 +10,7 @@ module Floe
 
         def initialize(options = {})
           require "awesome_spawn"
+          require "io/wait"
           require "tempfile"
 
           super
@@ -45,10 +46,63 @@ module Floe
           delete_secret(secrets_file)    if secrets_file
         end
 
+        def wait(timeout: nil, events: %i[create update delete], &block)
+          until_timestamp = Time.now.utc + timeout if timeout
+
+          r, w = IO.pipe
+
+          pid = AwesomeSpawn.run_detached(
+            self.class::DOCKER_COMMAND, :err => :out, :out => w, :params => wait_params(until_timestamp)
+          )
+
+          w.close
+
+          loop do
+            readable_timeout = until_timestamp - Time.now.utc if until_timestamp
+
+            # Wait for our end of the pipe to be readable and if it didn't timeout
+            # get the events from stdout
+            next if r.wait_readable(readable_timeout).nil?
+
+            # Get all events while the pipe is readable
+            notices = []
+            while r.ready?
+              notice = r.gets
+
+              # If the process has exited `r.gets` returns `nil` and the pipe is
+              # always `ready?`
+              break if notice.nil?
+
+              event, runner_context = parse_notice(notice)
+              next if event.nil? || !events.include?(event)
+
+              notices << [event, runner_context]
+            end
+
+            # If we're given a block yield the events otherwise return them
+            if block
+              notices.each(&block)
+            else
+              # Terminate the `docker events` process before returning the events
+              sigterm(pid)
+
+              return notices
+            end
+
+            # Check that the `docker events` process is still alive
+            Process.kill(0, pid)
+          rescue Errno::ESRCH
+            # Break out of the loop if the `docker events` process has exited
+            break
+          end
+        ensure
+          r.close
+        end
+
         def status!(runner_context)
           return if runner_context.key?("Error")
 
-          runner_context["container_state"] = inspect_container(runner_context["container_ref"]).first&.dig("State")
+          runner_context["container_state"] = inspect_container(runner_context["container_ref"])&.dig("State")
         end
 
         def running?(runner_context)
@@ -91,8 +145,45 @@ module Floe
           params << image
         end
 
+        def wait_params(until_timestamp)
+          params = ["events", [:format, "{{json .}}"], [:filter, "type=container"], [:since, Time.now.utc.to_i]]
+          params << [:until, until_timestamp.to_i] if until_timestamp
+          params
+        end
+
+        def parse_notice(notice)
+          notice = JSON.parse(notice)
+
+          status  = notice["status"]
+          event   = docker_event_status_to_event(status)
+          running = event != :delete
+
+          name, exit_code = notice.dig("Actor", "Attributes")&.values_at("name", "exitCode")
+
+          runner_context = {"container_ref" => name, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+          [event, runner_context]
+        rescue JSON::ParserError
+          []
+        end
+
+        def docker_event_status_to_event(status)
+          case status
+          when "create"
+            :create
+          when "start"
+            :update
+          when "die", "destroy"
+            :delete
+          else
+            :unkonwn
+          end
+        end
+
         def inspect_container(container_id)
-          JSON.parse(docker!("inspect", container_id).output)
+          JSON.parse(docker!("inspect", container_id).output).first
+        rescue
+          nil
         end
 
         def delete_container(container_id)
@@ -116,6 +207,12 @@ module Floe
           secrets_file.path
         end
 
+        def sigterm(pid)
+          Process.kill("TERM", pid)
+        rescue Errno::ESRCH
+          nil
+        end
+
         def global_docker_options
           []
         end

data/lib/floe/workflow/runner/docker_mixin.rb

@@ -6,9 +6,10 @@ module Floe
           image.match(%r{^(?<repository>.+/)?(?<image>.+):(?<tag>.+)$})&.named_captures&.dig("image")
         end
 
-        MAX_CONTAINER_NAME_SIZE = 63 - 5 - 9 # 63 is the max kubernetes pod name length
-                                             # -5 for the "floe-" prefix
-                                             # -9 for the random hex suffix and leading hyphen
+        # 63 is the max kubernetes pod name length
+        # -5 for the "floe-" prefix
+        # -9 for the random hex suffix and leading hyphen
+        MAX_CONTAINER_NAME_SIZE = 63 - 5 - 9
 
         def container_name(image)
           name = image_name(image)

data/lib/floe/workflow/runner/kubernetes.rb

@@ -53,7 +53,7 @@ module Floe
           name   = container_name(image)
           secret = create_secret!(secrets) if secrets && !secrets.empty?
 
-          runner_context = {"container_ref" => name, "secrets_ref" => secret}
+          runner_context = {"container_ref" => name, "container_state" => {"phase" => "Pending"}, "secrets_ref" => secret}
 
           begin
             create_pod!(name, image, env, secret)
@@ -102,6 +102,54 @@ module Floe
           delete_secret(secret) if secret
         end
 
+        def wait(timeout: nil, events: %i[create update delete])
+          retry_connection = true
+
+          begin
+            watcher = kubeclient.watch_pods(:namespace => namespace)
+
+            retry_connection = true
+
+            if timeout.to_i > 0
+              timeout_thread = Thread.new do
+                sleep(timeout)
+                watcher.finish
+              end
+            end
+
+            watcher.each do |notice|
+              break if error_notice?(notice)
+
+              event = kube_notice_type_to_event(notice.type)
+              next unless events.include?(event)
+
+              runner_context = parse_notice(notice)
+              next if runner_context.nil?
+
+              if block_given?
+                yield [event, runner_context]
+              else
+                timeout_thread&.kill # If we break out before the timeout, kill the timeout thread
+                return [[event, runner_context]]
+              end
+            end
+          rescue Kubeclient::HttpError => err
+            raise unless err.error_code == 401 && retry_connection
+
+            @kubeclient = nil
+            retry_connection = false
+            retry
+          ensure
+            begin
+              watch&.finish
+            rescue
+              nil
+            end
+
+            timeout_thread&.join(0)
+          end
+        end
+
         private
 
         attr_reader :ca_file, :kubeconfig_file, :kubeconfig_context, :namespace, :server, :token, :verify_ssl
@@ -116,7 +164,7 @@ module Floe
 
         def failed_container_states(context)
           container_statuses = context.dig("container_state", "containerStatuses") || []
-          container_statuses.map { |status| status["state"]&.values&.first }.compact
+          container_statuses.filter_map { |status| status["state"]&.values&.first }
                             .select { |state| FAILURE_REASONS.include?(state["reason"]) }
         end
 
@@ -217,6 +265,41 @@ module Floe
           nil
         end
 
+        def kube_notice_type_to_event(type)
+          case type
+          when "ADDED"
+            :create
+          when "MODIFIED"
+            :update
+          when "DELETED"
+            :delete
+          else
+            :unknown
+          end
+        end
+
+        def error_notice?(notice)
+          return false unless notice.type == "ERROR"
+
+          message = notice.object&.message
+          code    = notice.object&.code
+          reason  = notice.object&.reason
+
+          logger.warn("Received [#{code} #{reason}], [#{message}]")
+
+          true
+        end
+
+        def parse_notice(notice)
+          return if notice.object.nil?
+
+          pod             = notice.object
+          container_ref   = pod.metadata.name
+          container_state = pod.to_h[:status].deep_stringify_keys
+
+          {"container_ref" => container_ref, "container_state" => container_state}
+        end
+
         def kubeclient
           return @kubeclient unless @kubeclient.nil?
 

data/lib/floe/workflow/runner/podman.rb

@@ -55,6 +55,32 @@ module Floe
           nil
         end
 
+        def parse_notice(notice)
+          id, status, exit_code = JSON.parse(notice).values_at("ID", "Status", "ContainerExitCode")
+
+          event   = podman_event_status_to_event(status)
+          running = event != :delete
+
+          runner_context = {"container_ref" => id, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+          [event, runner_context]
+        rescue JSON::ParserError
+          []
+        end
+
+        def podman_event_status_to_event(status)
+          case status
+          when "create"
+            :create
+          when "init", "start"
+            :update
+          when "died", "cleanup", "remove"
+            :delete
+          else
+            :unknown
+          end
+        end
+
         alias podman! docker!
 
         def global_docker_options

data/lib/floe/workflow/runner.rb

@@ -5,29 +5,42 @@ module Floe
     class Runner
       include Logging
 
-      TYPES         = %w[docker podman kubernetes].freeze
       OUTPUT_MARKER = "__FLOE_OUTPUT__\n"
 
       def initialize(_options = {})
       end
 
+      @runners = {}
       class << self
-        attr_writer :docker_runner
+        # deprecated -- use Floe.set_runner instead
+        def docker_runner=(value)
+          set_runner("docker", value)
+        end
 
-        def docker_runner
-          @docker_runner ||= Floe::Workflow::Runner::Docker.new
+        # see Floe.set_runner
+        def set_runner(scheme, name_or_instance, options = {})
+          @runners[scheme] =
+            case name_or_instance
+            when "docker", nil
+              Floe::Workflow::Runner::Docker.new(options)
+            when "podman"
+              Floe::Workflow::Runner::Podman.new(options)
+            when "kubernetes"
+              Floe::Workflow::Runner::Kubernetes.new(options)
+            when Floe::Workflow::Runner
+              name_or_instance
+            else
+              raise ArgumentError, "docker runner must be one of: docker, podman, kubernetes"
+            end
         end
 
         def for_resource(resource)
           raise ArgumentError, "resource cannot be nil" if resource.nil?
 
+          # if no runners are set, default docker:// to docker
+          set_runner("docker", "docker") if @runners.empty?
           scheme = resource.split("://").first
-          case scheme
-          when "docker"
-            docker_runner
-          else
-            raise "Invalid resource scheme [#{scheme}]"
-          end
+          @runners[scheme] || raise(ArgumentError, "Invalid resource scheme [#{scheme}]")
         end
       end
 
@@ -55,6 +68,10 @@ module Floe
       def cleanup(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
+
+      def wait(timeout: nil, events: %i[create update delete])
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
     end
   end
 end

data/lib/floe/workflow/state.rb

@@ -33,12 +33,12 @@ module Floe
         raise Floe::InvalidWorkflowError, "State name [#{name}] must be less than or equal to 80 characters" if name.length > 80
       end
 
-      def wait(timeout: 5)
+      def wait(timeout: nil)
         start = Time.now.utc
 
         loop do
           return 0             if ready?
-          return Errno::EAGAIN if timeout.zero? || Time.now.utc - start > timeout
+          return Errno::EAGAIN if timeout && (timeout.zero? || Time.now.utc - start > timeout)
 
           sleep(1)
         end
@@ -58,7 +58,7 @@ module Floe
         context.state["Guid"]            = SecureRandom.uuid
         context.state["EnteredTime"]     = start_time
 
-        logger.info("Running state: [#{context.state_name}] with input [#{context.input}]...")
+        logger.info("Running state: [#{long_name}] with input [#{context.input}]...")
       end
 
       def finish
@@ -70,7 +70,8 @@ module Floe
         context.state["Duration"]       = finished_time - entered_time
         context.execution["EndTime"]    = finished_time_iso if context.next_state.nil?
 
-        logger.info("Running state: [#{context.state_name}] with input [#{context.input}]...Complete - next state: [#{context.next_state}] output: [#{context.output}]")
+        level = context.output&.[]("Error") ? :error : :info
+        logger.public_send(level, "Running state: [#{long_name}] with input [#{context.input}]...Complete #{context.next_state ? "- next state [#{context.next_state}]" : "workflow -"} output: [#{context.output}]")
 
         context.state_history << context.state
 
@@ -93,6 +94,18 @@ module Floe
         context.state.key?("FinishedTime")
       end
 
+      def waiting?
+        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
+      end
+
+      def wait_until
+        context.state["WaitUntil"] && Time.parse(context.state["WaitUntil"])
+      end
+
+      def long_name
+        "#{self.class.name.split("::").last}:#{name}"
+      end
+
       private
 
       def wait_until!(seconds: nil, time: nil)
@@ -105,10 +118,6 @@ module Floe
             time.iso8601
           end
       end
-
-      def waiting?
-        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
-      end
     end
   end
 end

data/lib/floe/workflow/states/choice.rb

@@ -18,14 +18,14 @@ module Floe
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
         end
 
-        def start(input)
-          super
-          input      = input_path.value(context, input)
+        def finish
+          input      = input_path.value(context, context.input)
           next_state = choices.detect { |choice| choice.true?(context, input) }&.next || default
           output     = output_path.value(context, input)
 
           context.next_state = next_state
           context.output     = output
+          super
         end
 
         def running?

data/lib/floe/workflow/states/fail.rb

@@ -15,18 +15,16 @@ module Floe
           @error_path = Path.new(payload["ErrorPath"]) if payload["ErrorPath"]
         end
 
-        def start(input)
-          super
+        def finish
           context.next_state = nil
           # TODO: support intrinsic functions here
           # see https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-fail-state.html
           #     https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-intrinsic-functions.html#asl-intrsc-func-generic
           context.output     = {
-            "Error" => @error_path ? @error_path.value(context, input) : error,
-            "Cause" => @cause_path ? @cause_path.value(context, input) : cause
+            "Error" => @error_path ? @error_path.value(context, context.input) : error,
+            "Cause" => @cause_path ? @cause_path.value(context, context.input) : cause
           }.compact
-          context.state["Error"] = context.output["Error"]
-          context.state["Cause"] = context.output["Cause"]
+          super
         end
 
         def running?

data/lib/floe/workflow/states/non_terminal_mixin.rb

@@ -4,6 +4,11 @@ module Floe
   class Workflow
     module States
       module NonTerminalMixin
+        def finish
+          context.next_state = end? ? nil : @next
+          super
+        end
+
         def validate_state_next!
           raise Floe::InvalidWorkflowError, "Missing \"Next\" field in state [#{name}]" if @next.nil? && !@end
           raise Floe::InvalidWorkflowError, "\"Next\" [#{@next}] not in \"States\" for state [#{name}]" if @next && !workflow.payload["States"].key?(@next)

data/lib/floe/workflow/states/pass.rb

@@ -24,13 +24,11 @@ module Floe
           validate_state!
         end
 
-        def start(input)
-          super
-
-          input = process_input(input)
-
+        def finish
+          input              = process_input(context.input)
           context.output     = process_output(input, result)
           context.next_state = end? ? nil : @next
+          super
         end
 
         def running?

data/lib/floe/workflow/states/succeed.rb

@@ -10,10 +10,10 @@ module Floe
           super
         end
 
-        def start(input)
-          super
+        def finish
           context.next_state = nil
-          context.output     = input
+          context.output     = context.input
+          super
         end
 
         def running?

data/lib/floe/workflow/states/task.rb

@@ -50,14 +50,14 @@ module Floe
 
           if success?
             output = parse_output(output)
-            context.state["Output"] = process_output(context.input.dup, output)
-            context.next_state      = next_state
+            context.output = process_output(context.input.dup, output)
+            super
           else
-            error = parse_error(output)
+            context.next_state = nil
+            context.output = error = parse_error(output)
+            super
             retry_state!(error) || catch_error!(error) || fail_workflow!(error)
           end
-
-          super
         ensure
           runner.cleanup(context.state["RunnerContext"])
         end
@@ -109,6 +109,7 @@ module Floe
 
           wait_until!(:seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
           context.next_state = context.state_name
+          logger.info("Running state: [#{long_name}] with input [#{context.input}]...Retry - delay: #{wait_until}")
           true
         end
 
@@ -118,13 +119,15 @@ module Floe
 
           context.next_state = catcher.next
           context.output     = catcher.result_path.set(context.input, error)
+          logger.info("Running state: [#{long_name}] with input [#{context.input}]...CatchError - next state: [#{context.next_state}] output: [#{context.output}]")
+
           true
         end
 
         def fail_workflow!(error)
           context.next_state     = nil
           context.output         = {"Error" => error["Error"], "Cause" => error["Cause"]}.compact
-          context.state["Error"] = context.output["Error"]
+          logger.error("Running state: [#{long_name}] with input [#{context.input}]...Complete workflow - output: [#{context.output}]")
         end
 
         def parse_error(output)

data/lib/floe/workflow/states/wait.rb

@@ -28,10 +28,8 @@ module Floe
 
         def start(input)
           super
-          input = input_path.value(context, input)
 
-          context.output     = output_path.value(context, input)
-          context.next_state = end? ? nil : @next
+          input = input_path.value(context, context.input)
 
           wait_until!(
             :seconds => seconds_path ? seconds_path.value(context, input).to_i : seconds,
@@ -39,6 +37,12 @@ module Floe
           )
         end
 
+        def finish
+          input          = input_path.value(context, context.input)
+          context.output = output_path.value(context, input)
+          super
+        end
+
         def running?
           waiting?
         end

data/lib/floe/workflow.rb

@@ -16,25 +16,76 @@ module Floe
         new(payload, context, credentials, name)
       end
 
-      def wait(workflows, timeout: 5)
+      def wait(workflows, timeout: nil, &block)
+        workflows = [workflows] if workflows.kind_of?(self)
         logger.info("checking #{workflows.count} workflows...")
 
-        start = Time.now.utc
-        ready = []
+        run_until   = Time.now.utc + timeout if timeout.to_i > 0
+        ready       = []
+        queue       = Queue.new
+        wait_thread = Thread.new do
+          loop do
+            Runner.for_resource("docker").wait do |event, runner_context|
+              queue.push([event, runner_context])
+            end
+          end
+        end
 
         loop do
           ready = workflows.select(&:step_nonblock_ready?)
-          break if timeout.zero? || Time.now.utc - start > timeout || !ready.empty?
-
-          sleep(1)
+          break if block.nil? && !ready.empty?
+
+          ready.each(&block)
+
+          # Break if all workflows are completed or we've exceeded the
+          # requested timeout
+          break if workflows.all?(&:end?)
+          break if timeout && (timeout.zero? || Time.now.utc > run_until)
+
+          # Find the earliest time that we should wakeup if no container events
+          # are caught, either a workflow in a Wait or Retry state or we've
+          # exceeded the requested timeout
+          wait_until = workflows.map(&:wait_until)
+                                .unshift(run_until)
+                                .compact
+                                .min
+
+          # If a workflow is in a waiting state wakeup the main thread when
+          # it will be done sleeping
+          if wait_until
+            sleep_thread = Thread.new do
+              sleep_duration = wait_until - Time.now.utc
+              sleep sleep_duration if sleep_duration > 0
+              queue.push(nil)
+            end
+          end
+
+          loop do
+            # Block until an event is raised
+            event, runner_context = queue.pop
+            break if event.nil?
+
+            # If the event is for one of our workflows set the updated runner_context
+            workflows.each do |workflow|
+              next unless workflow.context.state.dig("RunnerContext", "container_ref") == runner_context["container_ref"]
+
+              workflow.context.state["RunnerContext"] = runner_context
+            end
+
+            break if queue.empty?
+          end
+        ensure
+          sleep_thread&.kill
         end
 
         logger.info("checking #{workflows.count} workflows...Complete - #{ready.count} ready")
         ready
+      ensure
+        wait_thread&.kill
       end
     end
 
-    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at, :name
+    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at, :name, :comment
 
     def initialize(payload, context = nil, credentials = {}, name = nil)
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
@@ -49,6 +100,7 @@ module Floe
       @payload     = payload
       @context     = context
       @credentials = credentials || {}
+      @comment     = payload["Comment"]
       @start_at    = payload["StartAt"]
 
       @states         = payload["States"].to_a.map { |state_name, state| State.build!(self, state_name, state) }
@@ -74,7 +126,7 @@ module Floe
       current_state.run_nonblock!
     end
 
-    def step_nonblock_wait(timeout: 5)
+    def step_nonblock_wait(timeout: nil)
       current_state.wait(:timeout => timeout)
     end
 
@@ -82,6 +134,14 @@ module Floe
       current_state.ready?
     end
 
+    def waiting?
+      current_state.waiting?
+    end
+
+    def wait_until
+      current_state.wait_until
+    end
+
     def status
       context.status
     end

data/lib/floe.rb

@@ -45,7 +45,27 @@ module Floe
     @logger ||= NullLogger.new
   end
 
+  # Set the logger to use
+  #
+  # @example
+  #   require "logger"
+  #   Floe.logger = Logger.new($stdout)
+  #
+  # @param logger [Logger] logger to use for logging actions
   def self.logger=(logger)
     @logger = logger
   end
+
+  # Set the runner to use
+  #
+  # @example
+  #   Floe.set_runner "docker", kubernetes", {}
+  #   Floe.set_runner "docker", Floe::Workflow::Runner::Kubernetes.new({})
+  #
+  # @param scheme [String] scheme                           Protocol to register (e.g.: docker)
+  # @param name_or_instance [String|Floe::Workflow::Runner] Name of runner to use for docker (e.g.: docker)
+  # @param options [Hash]                                   Options for constructor of the runner (optional)
+  def self.set_runner(scheme, name_or_instance, options = {})
+    Floe::Workflow::Runner.set_runner(scheme, name_or_instance, options)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.10.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-01-17 00:00:00.000000000 Z
+date: 2024-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -16,14 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: io-wait
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: jsonpath
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,90 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: manageiq-style
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.21.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.21.2
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple Workflow Runner.
 email:
 executables: