floe 0.3.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ae01d21cee834a23c32b89a8a1418d5c02e1a7ec825305cce2ea783c9be6fc40
-  data.tar.gz: df5f35ef07d199e929e7cf3bb0a73620a6ee5def9b555abf1409a9f74c6f7b9a
+  metadata.gz: b2bab2c07fd71a45bcf896aa219993770c83eedd5c4e3268c318d8b9ba0413ed
+  data.tar.gz: 786064609be7fee8a19855cb97fd160984b5a8a7aedc85344eab01302155a988
 SHA512:
-  metadata.gz: e4dcf6ccfcf37b0cd187f4dfd876d60deec4f3a1037c20f76e4b759ae61cd394f68afcb23064c3a6c30f76f56e59ff038931e1630a5e7b01393702caf339d3fa
-  data.tar.gz: 0c588d1eb15f71e4ccd9870df09d72e0e1914f6d31d51b0275f39fb9a7fdd68930eb54f254b4dfa13cc184d0cf9b7babae671f7b2bfa5df1f87d77661b3e5036
+  metadata.gz: 45a9fd8a85bb5b1059574c0dc63b0434496b6c75634c567aba9f0008400867b4dbf8c33337b3940084c644606dc3fe6065fc4f79ac5732449ca0d9675d0e2531
+  data.tar.gz: f4147cd4e1633d2af02b7cbbd04735a9ffdc733359eb7f349b23a2fb8bada9de18a4f37fad82de543f2eb8060fa6fa52d1864a159f3ab3f780cbdab1824b6ab1

data/CHANGELOG.md

@@ -4,6 +4,22 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.4.1] - 2023-10-06
+### Added
+- Add Fail#CausePath and Fail#ErrorPath ([#110](https://github.com/ManageIQ/floe/pull/110))
+- Add Task#Retrier incremental backoff and Wait#Timestamp ([#100](https://github.com/ManageIQ/floe/pull/100))
+
+### Fixed
+- Combine stdout and stderr for docker and podman runners ([#104](https://github.com/ManageIQ/floe/pull/104))
+- Don't raise an exception on task failure ([#115](https://github.com/ManageIQ/floe/pull/115))
+- Fix task output handling ([#112](https://github.com/ManageIQ/floe/pull/112))
+- Fix Context#input not JSON parsed ([#122](https://github.com/ManageIQ/floe/pull/122))
+
+## [0.4.0] - 2023-09-26
+### Added
+- Add ability to run workflows asynchronously ([#52](https://github.com/ManageIQ/floe/pull/92))
+- Add Workflow.wait, Workflow#step_nonblock, Workflow#step_nonblock_wait ([#92](https://github.com/ManageIQ/floe/pull/92))
+
 ## [0.3.1] - 2023-08-29
 ### Added
 - Add more global podman runner options ([#90])(https://github.com/ManageIQ/floe/pull/90)
@@ -58,7 +74,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.3.1...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.4.1...HEAD
+[0.4.1]: https://github.com/ManageIQ/floe/compare/v0.4.0...v0.4.1
+[0.4.0]: https://github.com/ManageIQ/floe/compare/v0.3.1...v0.4.0
 [0.3.1]: https://github.com/ManageIQ/floe/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/ManageIQ/floe/compare/v0.2.3...v0.3.0
 [0.2.3]: https://github.com/ManageIQ/floe/compare/v0.2.2...v0.2.3

data/Gemfile

@@ -8,4 +8,8 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
 # Specify your gem's dependencies in floe.gemspec
 gemspec
 
+gem "manageiq-style"
 gem "rake", "~> 13.0"
+gem "rspec"
+gem "rubocop"
+gem "timecop"

data/README.md

@@ -56,7 +56,7 @@ bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"roleArn":
 ```ruby
 require 'floe'
 
-workflow = Floe::Workflow.load(File.read("workflow.asl"))
+workflow = Floe::Workflow.load("workflow.asl")
 workflow.run!
 ```
 
@@ -68,10 +68,51 @@ Floe::Workflow::Runner.docker_runner = Floe::Workflow::Runner::Podman.new
 # Or
 Floe::Workflow::Runner.docker_runner = Floe::Workflow::Runner::Kubernetes.new("namespace" => "default", "server" => "https://k8s.example.com:6443", "token" => "my-token")
 
-workflow = Floe::Workflow.load(File.read("workflow.asl"))
+workflow = Floe::Workflow.load("workflow.asl")
 workflow.run!
 ```
 
+### Non-Blocking Workflow Execution
+
+It is also possible to step through a workflow without blocking, and any state which
+would block will return `Errno::EAGAIN`.
+
+```ruby
+require 'floe'
+
+workflow = Floe::Workflow.load("workflow.asl")
+
+# Step through the workflow while it would not block
+workflow.run_nonblock
+
+# Go off and do some other task
+
+# Continue stepping until the workflow is finished
+workflow.run_nonblock
+```
+
+You can also use the `Floe::Workflow.wait` class method to wait on multiple workflows
+and return all that are ready to be stepped through.
+
+```ruby
+require 'floe'
+
+workflow1 = Floe::Workflow.load("workflow1.asl")
+workflow2 = Floe::Workflow.load("workflow2.asl")
+
+running_workflows = [workflow1, workflow2]
+until running_workflows.empty?
+  # Wait for any of the running workflows to be ready (up to the timeout)
+  ready_workflows = Floe::Workflow.wait(running_workflows)
+  # Step through the ready workflows until they would block
+  ready_workflows.each do |workflow|
+    loop while workflow.step_nonblock == 0
+  end
+  # Remove any finished workflows from the list of running_workflows
+  running_workflows.reject!(&:end?)
+end
+```
+
 ### Docker Runner Options
 
 #### Docker

data/exe/floe

@@ -18,9 +18,6 @@ Optimist.die(:docker_runner, "must be one of #{Floe::Workflow::Runner::TYPES.joi
 require "logger"
 Floe.logger = Logger.new($stdout)
 
-context = Floe::Workflow::Context.new(input: opts[:input])
-workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
-
 runner_klass = case opts[:docker_runner]
                when "docker"
                  Floe::Workflow::Runner::Docker
@@ -34,6 +31,9 @@ runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
 
 Floe::Workflow::Runner.docker_runner = runner_klass.new(runner_options)
 
+context = Floe::Workflow::Context.new(:input => opts[:input])
+workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
+
 workflow.run!
 
 puts workflow.output.inspect

data/floe.gemspec

@@ -34,8 +34,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "more_core_extensions"
   spec.add_dependency "optimist", "~>3.0"
-
-  spec.add_development_dependency "manageiq-style"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop"
 end

data/lib/floe/null_logger.rb

@@ -4,7 +4,7 @@ require 'logger'
 
 module Floe
   class NullLogger < Logger
-    def initialize(*)
+    def initialize(*) # rubocop:disable Lint/MissingSuper
     end
 
     def add(*_args, &_block)

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.3.1"
+  VERSION = "0.4.1"
 end

data/lib/floe/workflow/catcher.rb

@@ -10,7 +10,7 @@ module Floe
 
         @error_equals = payload["ErrorEquals"]
         @next         = payload["Next"]
-        @result_path  = payload.fetch("ResultPath", "$")
+        @result_path  = ReferencePath.new(payload.fetch("ResultPath", "$"))
       end
     end
   end

data/lib/floe/workflow/choice_rule/data.rb

@@ -51,27 +51,27 @@ module Floe
           raise "No such variable [#{variable}]" if value.nil? && !%w[IsNull IsPresent].include?(compare_key)
         end
 
-        def is_null?(value)
+        def is_null?(value) # rubocop:disable Naming/PredicateName
           value.nil?
         end
 
-        def is_present?(value)
+        def is_present?(value) # rubocop:disable Naming/PredicateName
           !value.nil?
         end
 
-        def is_numeric?(value)
+        def is_numeric?(value) # rubocop:disable Naming/PredicateName
           value.kind_of?(Integer) || value.kind_of?(Float)
         end
 
-        def is_string?(value)
+        def is_string?(value) # rubocop:disable Naming/PredicateName
           value.kind_of?(String)
         end
 
-        def is_boolean?(value)
+        def is_boolean?(value) # rubocop:disable Naming/PredicateName
           [true, false].include?(value)
         end
 
-        def is_timestamp?(value)
+        def is_timestamp?(value) # rubocop:disable Naming/PredicateName
           require "date"
 
           DateTime.rfc3339(value)

data/lib/floe/workflow/context.rb

@@ -5,6 +5,7 @@ module Floe
     class Context
       def initialize(context = nil, input: {})
         context = JSON.parse(context) if context.kind_of?(String)
+        input = JSON.parse(input) if input.kind_of?(String)
 
         @context = context || {
           "Execution"    => {
@@ -21,10 +22,58 @@ module Floe
         @context["Execution"]
       end
 
+      def started?
+        execution.key?("StartTime")
+      end
+
+      def running?
+        started? && !ended?
+      end
+
+      def ended?
+        execution.key?("EndTime")
+      end
+
       def state
         @context["State"]
       end
 
+      def input
+        state["Input"]
+      end
+
+      def output
+        state["Output"]
+      end
+
+      def output=(val)
+        state["Output"] = val
+      end
+
+      def state_name
+        state["Name"]
+      end
+
+      def next_state
+        state["NextState"]
+      end
+
+      def next_state=(val)
+        state["NextState"] = val
+      end
+
+      def status
+        if !started?
+          "pending"
+        elsif running?
+          "running"
+        elsif state["Error"]
+          "failure"
+        else
+          "success"
+        end
+      end
+
       def state=(val)
         @context["State"] = val
       end

data/lib/floe/workflow/runner/docker.rb

@@ -18,34 +18,103 @@ module Floe
 
           image = resource.sub("docker://", "")
 
-          params  = ["run", :rm]
-          params += [[:net, "host"]] if network == "host"
-          params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
-
           secrets_file = nil
+          if secrets && !secrets.empty?
+            secrets_file = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets"
+          end
+
+          output = run_container(image, env, secrets_file)
+
+          {"exit_code" => 0, "output" => output}
+        ensure
+          cleanup({"secrets_ref" => secrets_file})
+        end
+
+        def run_async!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image = resource.sub("docker://", "")
+
+          runner_context = {}
 
           if secrets && !secrets.empty?
-            secrets_file = Tempfile.new
-            secrets_file.write(secrets.to_json)
-            secrets_file.flush
+            runner_context["secrets_ref"] = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets"
+          end
 
-            params << [:e, "_CREDENTIALS=/run/secrets"]
-            params << [:v, "#{secrets_file.path}:/run/secrets:z"]
+          begin
+            runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"], :detached => true)
+          rescue
+            cleanup(runner_context)
+            raise
           end
 
-          params << image
+          runner_context
+        end
 
-          logger.debug("Running docker: #{AwesomeSpawn.build_command_line("docker", params)}")
-          result = AwesomeSpawn.run!("docker", :params => params)
+        def cleanup(runner_context)
+          container_id, secrets_file = runner_context.values_at("container_ref", "secrets_ref")
 
-          [result.exit_status, result.output]
-        ensure
-          secrets_file&.close!
+          delete_container(container_id) if container_id
+          File.unlink(secrets_file)      if secrets_file && File.exist?(secrets_file)
+        end
+
+        def status!(runner_context)
+          runner_context["container_state"] = inspect_container(runner_context["container_ref"]).first&.dig("State")
+        end
+
+        def running?(runner_context)
+          runner_context.dig("container_state", "Running")
+        end
+
+        def success?(runner_context)
+          runner_context.dig("container_state", "ExitCode") == 0
+        end
+
+        def output(runner_context)
+          output = docker!("logs", runner_context["container_ref"], :combined_output => true).output
+          runner_context["output"] = output
         end
 
         private
 
         attr_reader :network
+
+        def run_container(image, env, secrets_file, detached: false)
+          params  = ["run"]
+          params << (detached ? :detach : :rm)
+          params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+          params << [:net, "host"] if @network == "host"
+          params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
+          params << image
+
+          logger.debug("Running docker: #{AwesomeSpawn.build_command_line("docker", params)}")
+
+          result = docker!(*params)
+          result.output
+        end
+
+        def inspect_container(container_id)
+          JSON.parse(docker!("inspect", container_id).output)
+        end
+
+        def delete_container(container_id)
+          docker!("rm", container_id)
+        rescue
+          nil
+        end
+
+        def create_secret(secrets)
+          secrets_file = Tempfile.new
+          secrets_file.write(secrets.to_json)
+          secrets_file.close
+          secrets_file.path
+        end
+
+        def docker!(*params, **kwargs)
+          AwesomeSpawn.run!("docker", :params => params, **kwargs)
+        end
       end
     end
   end

data/lib/floe/workflow/runner/kubernetes.rb

@@ -46,27 +46,65 @@ module Floe
           secret = create_secret!(secrets) if secrets && !secrets.empty?
 
           begin
+            runner_context = {"container_ref" => name}
+
             create_pod!(name, image, env, secret)
             loop do
               case pod_info(name).dig("status", "phase")
               when "Pending", "Running"
                 sleep(1)
-              when "Succeeded"
-                return [0, output(name)]
-              else
-                return [1, output(name)]
+              else # also "Succeeded"
+                runner_context["exit_code"] = 0
+                output(runner_context)
+                break
               end
             end
+
+            runner_context
           ensure
-            cleanup(name, secret)
+            cleanup({"container_ref" => name, "secrets_ref" => secret})
+          end
+        end
+
+        def run_async!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image  = resource.sub("docker://", "")
+          name   = pod_name(image)
+          secret = create_secret!(secrets) if secrets && !secrets.empty?
+
+          runner_context = {"container_ref" => name, "secrets_ref" => secret}
+
+          begin
+            create_pod!(name, image, env, secret)
+          rescue
+            cleanup(runner_context)
+            raise
           end
+
+          runner_context
+        end
+
+        def status!(runner_context)
+          runner_context["container_state"] = pod_info(runner_context["container_ref"])["status"]
         end
 
-        def output(pod)
-          kubeclient.get_pod_log(pod, namespace).body
+        def running?(runner_context)
+          %w[Pending Running].include?(runner_context.dig("container_state", "phase"))
         end
 
-        def cleanup(pod, secret)
+        def success?(runner_context)
+          runner_context.dig("container_state", "phase") == "Succeeded"
+        end
+
+        def output(runner_context)
+          output = kubeclient.get_pod_log(runner_context["container_ref"], namespace).body
+          runner_context["output"] = output
+        end
+
+        def cleanup(runner_context)
+          pod, secret = runner_context.values_at("container_ref", "secrets_ref")
+
           delete_pod(pod)       if pod
           delete_secret(secret) if secret
         end

data/lib/floe/workflow/runner/podman.rb

@@ -31,29 +31,99 @@ module Floe
 
           image = resource.sub("docker://", "")
 
-          params  = ["run", :rm]
-          params += [[:net, "host"]] if @network == "host"
-          params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
+          if secrets && !secrets.empty?
+            secret = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets/#{secret}"
+          end
+
+          output = run_container(image, env, secret)
+
+          {"exit_code" => 0, :output => output}
+        ensure
+          delete_secret(secret) if secret
+        end
+
+        def run_async!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image = resource.sub("docker://", "")
 
           if secrets && !secrets.empty?
-            secret_guid = SecureRandom.uuid
-            podman!("secret", "create", secret_guid, "-", :in_data => secrets.to_json)
+            secret_guid = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets/#{secret_guid}"
+          end
 
-            params << [:e, "_CREDENTIALS=/run/secrets/#{secret_guid}"]
-            params << [:secret, secret_guid]
+          begin
+            container_id = run_container(image, env, secret_guid, :detached => true)
+          rescue
+            cleanup({"container_ref" => container_id, "secrets_ref" => secret_guid})
+            raise
           end
 
+          {"container_ref" => container_id, "secrets_ref" => secret_guid}
+        end
+
+        def cleanup(runner_context)
+          container_id, secret_guid = runner_context.values_at("container_ref", "secrets_ref")
+
+          delete_container(container_id) if container_id
+          delete_secret(secret_guid)     if secret_guid
+        end
+
+        def status!(runner_context)
+          runner_context["container_state"] = inspect_container(runner_context["container_ref"]).first&.dig("State")
+        end
+
+        def running?(runner_context)
+          runner_context.dig("container_state", "Running")
+        end
+
+        def success?(runner_context)
+          runner_context.dig("container_state", "ExitCode") == 0
+        end
+
+        def output(runner_context)
+          output = podman!("logs", runner_context["container_ref"], :combined_output => true).output
+          runner_context["output"] = output
+        end
+
+        private
+
+        def run_container(image, env, secret, detached: false)
+          params  = ["run"]
+          params << (detached ? :detach : :rm)
+          params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+          params << [:net, "host"] if @network == "host"
+          params << [:secret, secret] if secret
           params << image
 
           logger.debug("Running podman: #{AwesomeSpawn.build_command_line("podman", params)}")
+
           result = podman!(*params)
+          result.output
+        end
 
-          [result.exit_status, result.output]
-        ensure
-          AwesomeSpawn.run("podman", :params => ["secret", "rm", secret_guid]) if secret_guid
+        def inspect_container(container_id)
+          JSON.parse(podman!("inspect", container_id).output)
         end
 
-        private
+        def delete_container(container_id)
+          podman!("rm", container_id)
+        rescue
+          nil
+        end
+
+        def create_secret(secrets)
+          secret_guid = SecureRandom.uuid
+          podman!("secret", "create", secret_guid, "-", :in_data => secrets.to_json)
+          secret_guid
+        end
+
+        def delete_secret(secret_guid)
+          podman!("secret", "rm", secret_guid)
+        rescue
+          nil
+        end
 
         def podman!(*args, **kwargs)
           params = podman_global_options + args

data/lib/floe/workflow/runner.rb

@@ -5,7 +5,8 @@ module Floe
     class Runner
       include Logging
 
-      TYPES = %w[docker podman kubernetes].freeze
+      TYPES         = %w[docker podman kubernetes].freeze
+      OUTPUT_MARKER = "__FLOE_OUTPUT__\n"
 
       def initialize(_options = {})
       end
@@ -30,7 +31,28 @@ module Floe
         end
       end
 
-      def run!(image, env = {}, secrets = {})
+      def run!(resource, env = {}, secrets = {})
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+
+      # @return [Hash] runner_context
+      def run_async!(_image, _env = {}, _secrets = {})
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+
+      def running?(_runner_context)
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+
+      def success?(_runner_context)
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+
+      def output(_runner_context)
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+
+      def cleanup(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
     end

data/lib/floe/workflow/state.rb

@@ -29,9 +29,86 @@ module Floe
         @comment  = payload["Comment"]
       end
 
+      def run!(_input = nil)
+        run_wait until run_nonblock! == 0
+      end
+
+      def run_wait(timeout: 5)
+        start = Time.now.utc
+
+        loop do
+          return 0             if ready?
+          return Errno::EAGAIN if timeout.zero? || Time.now.utc - start > timeout
+
+          sleep(1)
+        end
+      end
+
+      def run_nonblock!
+        start(context.input) unless started?
+        return Errno::EAGAIN unless ready?
+
+        finish
+      end
+
+      def start(_input)
+        start_time = Time.now.utc.iso8601
+
+        context.execution["StartTime"] ||= start_time
+        context.state["Guid"]            = SecureRandom.uuid
+        context.state["EnteredTime"]     = start_time
+
+        logger.info("Running state: [#{context.state_name}] with input [#{context.input}]...")
+      end
+
+      def finish
+        finished_time     = Time.now.utc
+        finished_time_iso = finished_time.iso8601
+        entered_time      = Time.parse(context.state["EnteredTime"])
+
+        context.state["FinishedTime"] ||= finished_time_iso
+        context.state["Duration"]       = finished_time - entered_time
+        context.execution["EndTime"]    = finished_time_iso if context.next_state.nil?
+
+        logger.info("Running state: [#{context.state_name}] with input [#{context.input}]...Complete - next state: [#{context.next_state}] output: [#{context.output}]")
+
+        context.state_history << context.state
+
+        0
+      end
+
       def context
         workflow.context
       end
+
+      def started?
+        context.state.key?("EnteredTime")
+      end
+
+      def ready?
+        !started? || !running?
+      end
+
+      def finished?
+        context.state.key?("FinishedTime")
+      end
+
+      private
+
+      def wait(seconds: nil, time: nil)
+        context.state["WaitUntil"] =
+          if seconds
+            (Time.parse(context.state["EnteredTime"]) + seconds).iso8601
+          elsif time.kind_of?(String)
+            time
+          else
+            time.iso8601
+          end
+      end
+
+      def waiting?
+        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
+      end
     end
   end
 end

data/lib/floe/workflow/states/choice.rb

@@ -16,16 +16,18 @@ module Floe
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
         end
 
-        def run!(input)
+        def start(input)
+          super
           input      = input_path.value(context, input)
           next_state = choices.detect { |choice| choice.true?(context, input) }&.next || default
           output     = output_path.value(context, input)
 
-          [next_state, output]
+          context.next_state = next_state
+          context.output     = output
         end
 
-        def status
-          "running"
+        def running?
+          false
         end
 
         def end?

data/lib/floe/workflow/states/fail.rb

@@ -9,16 +9,28 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          @cause = payload["Cause"]
-          @error = payload["Error"]
+          @cause      = payload["Cause"]
+          @error      = payload["Error"]
+          @cause_path = Path.new(payload["CausePath"]) if payload["CausePath"]
+          @error_path = Path.new(payload["ErrorPath"]) if payload["ErrorPath"]
         end
 
-        def run!(input)
-          [nil, input]
+        def start(input)
+          super
+          context.next_state = nil
+          # TODO: support intrinsic functions here
+          # see https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-fail-state.html
+          #     https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-intrinsic-functions.html#asl-intrsc-func-generic
+          context.output     = {
+            "Error" => @error_path ? @error_path.value(context, input) : error,
+            "Cause" => @cause_path ? @cause_path.value(context, input) : cause
+          }.compact
+          context.state["Error"] = context.output["Error"]
+          context.state["Cause"] = context.output["Cause"]
         end
 
-        def status
-          "errored"
+        def running?
+          false
         end
 
         def end?

data/lib/floe/workflow/states/map.rb

@@ -5,6 +5,7 @@ module Floe
     module States
       class Map < Floe::Workflow::State
         def initialize(*)
+          super
           raise NotImplementedError
         end
       end

data/lib/floe/workflow/states/parallel.rb

@@ -5,6 +5,7 @@ module Floe
     module States
       class Parallel < Floe::Workflow::State
         def initialize(*)
+          super
           raise NotImplementedError
         end
       end

data/lib/floe/workflow/states/pass.rb

@@ -19,16 +19,18 @@ module Floe
           @result_path = ReferencePath.new(payload.fetch("ResultPath", "$"))
         end
 
-        def run!(input)
+        def start(input)
+          super
           output = input_path.value(context, input)
           output = result_path.set(output, result) if result && result_path
           output = output_path.value(context, output)
 
-          [@end ? nil : @next, output]
+          context.next_state = end? ? nil : @next
+          context.output     = output
         end
 
-        def status
-          @end ? "success" : "running"
+        def running?
+          false
         end
 
         def end?

data/lib/floe/workflow/states/succeed.rb

@@ -10,12 +10,14 @@ module Floe
           super
         end
 
-        def run!(input)
-          [nil, input]
+        def start(input)
+          super
+          context.next_state = nil
+          context.output     = input
         end
 
-        def status
-          "success"
+        def running?
+          false
         end
 
         def end?

data/lib/floe/workflow/states/task.rb

@@ -15,6 +15,7 @@ module Floe
           @next              = payload["Next"]
           @end               = !!payload["End"]
           @resource          = payload["Resource"]
+          @runner            = Floe::Workflow::Runner.for_resource(@resource)
           @timeout_seconds   = payload["TimeoutSeconds"]
           @retry             = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
           @catch             = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
@@ -26,36 +27,65 @@ module Floe
           @credentials       = PayloadTemplate.new(payload["Credentials"])    if payload["Credentials"]
         end
 
-        def run!(input)
-          input = input_path.value(context, input)
-          input = parameters.value(context, input) if parameters
-
-          runner = Floe::Workflow::Runner.for_resource(resource)
-          _exit_status, results = runner.run!(resource, input, credentials&.value({}, workflow.credentials))
-
-          output = process_output!(input, results)
-          [@end ? nil : @next, output]
-        rescue => err
-          retrier = self.retry.detect { |r| (r.error_equals & [err.to_s, "States.ALL"]).any? }
-          retry if retry!(retrier)
+        def start(input)
+          super
 
-          catcher = self.catch.detect { |c| (c.error_equals & [err.to_s, "States.ALL"]).any? }
-          raise if catcher.nil?
+          input          = process_input(input)
+          runner_context = runner.run_async!(resource, input, credentials&.value({}, workflow.credentials))
 
-          [catcher.next, output]
+          context.state["RunnerContext"] = runner_context
         end
 
         def status
           @end ? "success" : "running"
         end
 
+        def finish
+          output = runner.output(context.state["RunnerContext"])
+
+          if success?
+            output = parse_output(output)
+            context.state["Output"] = process_output!(output)
+            context.next_state      = next_state
+          else
+            error = parse_error(output)
+            retry_state!(error) || catch_error!(error) || fail_workflow!(error)
+          end
+
+          super
+        ensure
+          runner.cleanup(context.state["RunnerContext"])
+        end
+
+        def running?
+          return true if waiting?
+
+          runner.status!(context.state["RunnerContext"])
+          runner.running?(context.state["RunnerContext"])
+        end
+
         def end?
           @end
         end
 
         private
 
-        def retry!(retrier)
+        attr_reader :runner
+
+        def success?
+          runner.success?(context.state["RunnerContext"])
+        end
+
+        def find_retrier(error)
+          self.retry.detect { |r| (r.error_equals & [error, "States.ALL"]).any? }
+        end
+
+        def find_catcher(error)
+          self.catch.detect { |c| (c.error_equals & [error, "States.ALL"]).any? }
+        end
+
+        def retry_state!(error)
+          retrier = find_retrier(error["Error"]) if error
           return if retrier.nil?
 
           # If a different retrier is hit reset the context
@@ -68,24 +98,61 @@ module Floe
 
           return if context["State"]["RetryCount"] > retrier.max_attempts
 
-          Kernel.sleep(retrier.sleep_duration(context["State"]["RetryCount"]))
+          wait(:seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
+          context.next_state = context.state_name
           true
         end
 
-        def process_output!(output, results)
+        def catch_error!(error)
+          catcher = find_catcher(error["Error"]) if error
+          return if catcher.nil?
+
+          context.next_state = catcher.next
+          context.output     = catcher.result_path.set(context.input, error)
+          true
+        end
+
+        def fail_workflow!(error)
+          context.next_state     = nil
+          context.output         = {"Error" => error["Error"], "Cause" => error["Cause"]}.compact
+          context.state["Error"] = context.output["Error"]
+        end
+
+        def process_input(input)
+          input = input_path.value(context, input)
+          input = parameters.value(context, input) if parameters
+          input
+        end
+
+        def parse_error(output)
+          return if output.nil?
+
+          JSON.parse(output)
+        rescue JSON::ParserError
+          {"Error" => output}
+        end
+
+        def parse_output(output)
+          return if output.nil?
+
+          JSON.parse(output.split("\n").last)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def process_output!(results)
+          output = context.input.dup
           return output if results.nil?
           return if output_path.nil?
 
-          begin
-            results = JSON.parse(results)
-          rescue JSON::ParserError
-            results = {"results" => results}
-          end
-
           results = result_selector.value(context, results) if result_selector
           output  = result_path.set(output, results)
           output_path.value(context, output)
         end
+
+        def next_state
+          end? ? nil : @next
+        end
       end
     end
   end

data/lib/floe/workflow/states/wait.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module Floe
   class Workflow
     module States
@@ -9,28 +11,42 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          @next    = payload["Next"]
-          @end     = !!payload["End"]
-          @seconds = payload["Seconds"].to_i
+          @next           = payload["Next"]
+          @end            = !!payload["End"]
+          @seconds        = payload["Seconds"]&.to_i
+          @timestamp      = payload["Timestamp"]
+          @timestamp_path = Path.new(payload["TimestampPath"]) if payload.key?("TimestampPath")
+          @seconds_path   = Path.new(payload["SecondsPath"]) if payload.key?("SecondsPath")
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
         end
 
-        def run!(input)
+        def start(input)
+          super
           input = input_path.value(context, input)
-          sleep(seconds)
-          output = output_path.value(context, input)
-          [@end ? nil : @next, output]
+
+          context.output     = output_path.value(context, input)
+          context.next_state = end? ? nil : @next
+          please_hold(input)
         end
 
-        def status
-          @end ? "success" : "running"
+        def running?
+          waiting?
         end
 
         def end?
           @end
         end
+
+        private
+
+        def please_hold(input)
+          wait(
+            :seconds => @seconds_path ? @seconds_path.value(context, input).to_i : @seconds,
+            :time    => @timestamp_path ? @timestamp_path.value(context, input) : @timestamp
+          )
+        end
       end
     end
   end

data/lib/floe/workflow.rb

@@ -12,9 +12,26 @@ module Floe
         payload = path_or_io.respond_to?(:read) ? path_or_io.read : File.read(path_or_io)
         new(payload, context, credentials)
       end
+
+      def wait(workflows, timeout: 5)
+        logger.info("checking #{workflows.count} workflows...")
+
+        start = Time.now.utc
+        ready = []
+
+        loop do
+          ready = workflows.select(&:step_nonblock_ready?)
+          break if timeout.zero? || Time.now.utc - start > timeout || !ready.empty?
+
+          sleep(1)
+        end
+
+        logger.info("checking #{workflows.count} workflows...Complete - #{ready.count} ready")
+        ready
+      end
     end
 
-    attr_reader :context, :credentials, :output, :payload, :states, :states_by_name, :current_state, :status
+    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at
 
     def initialize(payload, context = nil, credentials = {})
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
@@ -24,62 +41,69 @@ module Floe
       @payload     = payload
       @context     = context
       @credentials = credentials
+      @start_at    = payload["StartAt"]
 
       @states         = payload["States"].to_a.map { |name, state| State.build!(self, name, state) }
       @states_by_name = @states.each_with_object({}) { |state, result| result[state.name] = state }
-      start_at        = @payload["StartAt"]
-
-      context.state["Name"] ||= start_at
-
-      current_state_name = context.state["Name"]
-      @current_state     = @states_by_name[current_state_name]
 
-      @status = current_state_name == start_at ? "pending" : current_state.status
+      unless context.state.key?("Name")
+        context.state["Name"] = start_at
+        context.state["Input"] = context.execution["Input"].dup
+      end
     rescue JSON::ParserError => err
       raise Floe::InvalidWorkflowError, err.message
     end
 
-    def step
-      @status = "running" if @status == "pending"
-      context.execution["StartTime"] ||= Time.now.utc
-
-      context.state["Guid"]    = SecureRandom.uuid
-      context.state["Input"] ||= context.execution["Input"].dup
+    def run!
+      step until end?
+      self
+    end
 
-      logger.info("Running state: [#{current_state.name}] with input [#{context.state["Input"]}]...")
+    def step
+      step_nonblock_wait until step_nonblock == 0
+      self
+    end
 
-      context.state["EnteredTime"] = Time.now.utc
+    def run_nonblock
+      loop while step_nonblock == 0 && !end?
+      self
+    end
 
-      tick = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      next_state, output = current_state.run!(context.state["Input"])
-      tock = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    def step_nonblock
+      return Errno::EPERM if end?
 
-      context.state["FinishedTime"] = Time.now.utc
-      context.state["Duration"]     = tock - tick
-      context.state["Output"]       = output
+      step_next
+      current_state.run_nonblock!
+    end
 
-      logger.info("Running state: [#{current_state.name}] with input [#{context["Input"]}]...Complete - next state: [#{next_state}] output: [#{output}]")
+    def step_nonblock_wait(timeout: 5)
+      current_state.run_wait(:timeout => timeout)
+    end
 
-      context.state_history << context.state
+    def step_nonblock_ready?
+      current_state.ready?
+    end
 
-      @status        = current_state.status
-      @current_state = next_state && @states_by_name[next_state]
-      @output        = output if end?
+    def status
+      context.status
+    end
 
-      context.state = {"Name" => next_state, "Input" => output} unless end?
+    def output
+      context.output if end?
+    end
 
-      self
+    def end?
+      context.ended?
     end
 
-    def run!
-      until end?
-        step
-      end
-      self
+    def current_state
+      @states_by_name[context.state_name]
     end
 
-    def end?
-      current_state.nil?
+    private
+
+    def step_next
+      context.state = {"Name" => context.next_state, "Input" => context.output} if context.next_state
     end
   end
 end

data/lib/floe.rb

@@ -32,6 +32,7 @@ require_relative "floe/workflow/states/task"
 require_relative "floe/workflow/states/wait"
 
 require "jsonpath"
+require "time"
 
 module Floe
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.1
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-08-29 00:00:00.000000000 Z
+date: 2023-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -80,48 +80,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: manageiq-style
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 description: Simple Workflow Runner.
 email:
 executables: