floe 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0e0b86c08d322e2ebc8e115e5193d0493f8257c831f7377b18c32d4284ae6f2
-  data.tar.gz: 1bf5ed62abafe2e1af6025dbaf47d9faa3b1c93ffec8fd88fbeacd4445efc795
+  metadata.gz: 75d5be2f5b9cdcfc64b4b3994d32b9e3955486fd99bf4e05c62c2414485188c8
+  data.tar.gz: 7257009d5942f157f2ccef6499c77e1f9033e0ac4846a08f3f589943bd4794ef
 SHA512:
-  metadata.gz: 34f64555f2472c121fcea9dda8baac7fbe853cdbc5e949c831d7b7f070878d3ef2f86ed90d907b5a7033539fcad6aeea64a4dee6475a752d7f89e4f679a4b269
-  data.tar.gz: 378c5bcd562d4bbeb7e3eb13eb55b1d5d8e7aed0ea4749c2b65f951cf3be0e5f13d3e288cdbf63b4a5729e98c5010656d6dca3db60ffbf8463988008980edd83
+  metadata.gz: bd9275c7e845841fc472e3e0ec92593354c6293bae3c8dc9d7258acb71fcd5db7abaca2c2aadcc4c7701952b65eae8a14da29c1e8e26d71a7d9df2c6aad9abeb
+  data.tar.gz: 8cc089f45244d80428d92feeeb162d8c65fafcea915e00405c54f5340dee4f9106113ad87e36c97a18fd159c3d338bee0f78869e91d200f19b145df494e5edfd

data/.codeclimate.yml

@@ -1,3 +1,4 @@
+version: '2'
 prepare:
   fetch:
   - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
@@ -8,9 +9,28 @@ prepare:
     path: styles/base.yml
   - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
     path: styles/cc_base.yml
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
 plugins:
   rubocop:
     enabled: true
     config: ".rubocop_cc.yml"
     channel: rubocop-1-56-3
-version: '2'
+exclude_patterns:
+- spec/

data/CHANGELOG.md

@@ -4,6 +4,24 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.15.0] - 2024-10-28
+### Added
+- Add WorkflowBase base class for Workflow ([#279](https://github.com/ManageIQ/floe/pull/279))
+- Add tool for using the aws stepfunctions simulator ([#244](https://github.com/ManageIQ/floe/pull/244))
+- Implement Map state ([#184](https://github.com/ManageIQ/floe/pull/184))
+- Add Map State Tolerated Failure ([#282](https://github.com/ManageIQ/floe/pull/282))
+- Run Map iterations in parallel up to MaxConcurrency ([#283](https://github.com/ManageIQ/floe/pull/283))
+- Implement Parallel State ([#291](https://github.com/ManageIQ/floe/pull/291))
+
+### Changed
+- More granular compare_key and determine path at initialization time ([#274](https://github.com/ManageIQ/floe/pull/274))
+- For Choice validation, use instance variables and not payload ([#277](https://github.com/ManageIQ/floe/pull/277))
+- Return ExceedToleratedFailureThreshold if ToleratedFailureCount/Percentage is present ([#285](https://github.com/ManageIQ/floe/pull/285))
+
+### Fixed
+- Fix case on log messages ([#280](https://github.com/ManageIQ/floe/pull/280))
+- Handle either ToleratedFailureCount or ToleratedFailurePercentage ([#284](https://github.com/ManageIQ/floe/pull/284))
+
 ## [0.14.0] - 2024-08-20
 ### Added
 - Implement "IsNumeric": false ([#266](https://github.com/ManageIQ/floe/pull/266))

data/README.md

@@ -197,6 +197,14 @@ Options supported by the kubernetes docker runner are:
 * `ca_file` - Path to a certificate-authority file for the kubernetes API, only valid if server and token are passed.  If present `/run/secrets/kubernetes.io/serviceaccount/ca.crt` will be used
 * `verify_ssl` - Controls if the kubernetes API certificate-authority should be verified, defaults to "true", only vaild if server and token are passed
 
+## Features Not Yet Supported
+
+The following are not yet supported:
+- Map State Fields:
+  - ItemReader
+  - ItemSelector/ItemBatcher
+  - ResultWriter
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/examples/map.asl

@@ -0,0 +1,46 @@
+{
+  "Comment": "Using Map state in Inline mode",
+  "StartAt": "Pass",
+  "States": {
+    "Pass": {
+      "Type": "Pass",
+      "Next": "Map demo",
+      "Result": {
+        "foo": "bar",
+        "colors": [
+          "red",
+          "green",
+          "blue",
+          "yellow",
+          "white"
+        ]
+      }
+    },
+    "Map demo": {
+      "Type": "Map",
+      "ItemsPath": "$.colors",
+      "MaxConcurrency": 2,
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "INLINE"
+        },
+        "StartAt": "Generate UUID",
+        "States": {
+          "Generate UUID": {
+            "Type": "Pass",
+            "Next": "Sleep",
+            "Parameters": {
+              "uuid.$": "States.UUID()"
+            }
+          },
+          "Sleep": {
+            "Type": "Task",
+            "Resource": "docker://docker.io/agrare/sleep:latest",
+            "End": true
+          }
+        }
+      },
+      "End": true
+    }
+  }
+}

data/examples/parallel.asl

@@ -0,0 +1,32 @@
+{
+  "Comment": "Parallel Example.",
+  "StartAt": "FunWithMath",
+  "States": {
+    "FunWithMath": {
+      "Type": "Parallel",
+      "End": true,
+      "Branches": [
+        {
+          "StartAt": "Add",
+          "States": {
+            "Add": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        },
+        {
+          "StartAt": "Subtract",
+          "States": {
+            "Subtract": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        }
+      ]
+    }
+  }
+}

data/lib/floe/cli.rb

@@ -13,17 +13,11 @@ module Floe
     def run(args = ARGV)
       workflows_inputs, opts = parse_options!(args)
 
-      credentials =
-        if opts[:credentials_given]
-          opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
-        elsif opts[:credentials_file_given]
-          File.read(opts[:credentials_file])
-        end
+      credentials = create_credentials(opts)
 
       workflows =
         workflows_inputs.each_slice(2).map do |workflow, input|
-          context = Floe::Workflow::Context.new(opts[:context], :input => input, :credentials => credentials)
-          Floe::Workflow.load(workflow, context)
+          create_workflow(workflow, opts[:context], input, credentials)
         end
 
       Floe::Workflow.wait(workflows, &:run_nonblock)
@@ -82,5 +76,18 @@ module Floe
 
       return workflows_inputs, opts
     end
+
+    def create_credentials(opts)
+      if opts[:credentials_given]
+        opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
+      elsif opts[:credentials_file_given]
+        File.read(opts[:credentials_file])
+      end
+    end
+
+    def create_workflow(workflow, context_payload, input, credentials)
+      context = Floe::Workflow::Context.new(context_payload, :input => input, :credentials => credentials)
+      Floe::Workflow.load(workflow, context)
+    end
   end
 end

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.14.0"
+  VERSION = "0.15.0"
 end

data/lib/floe/workflow/branch.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class Branch < Floe::WorkflowBase
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/data.rb

@@ -4,16 +4,20 @@ module Floe
   class Workflow
     class ChoiceRule
       class Data < Floe::Workflow::ChoiceRule
-        COMPARE_KEYS = %w[IsNull IsPresent IsNumeric IsString IsBoolean IsTimestamp String Numeric Boolean Timestamp].freeze
+        TYPES      = ["String", "Numeric", "Boolean", "Timestamp", "Present", "Null"].freeze
+        COMPARES   = ["Equals", "LessThan", "GreaterThan", "LessThanEquals", "GreaterThanEquals", "Matches"].freeze
+        # e.g.: (Is)(String), (Is)(Present)
+        TYPE_CHECK = /^(Is)(#{TYPES.join("|")})$/.freeze
+        # e.g.: (String)(LessThan)(Path), (Numeric)(GreaterThanEquals)()
+        OPERATION  = /^(#{(TYPES - %w[Null Present]).join("|")})(#{COMPARES.join("|")})(Path)?$/.freeze
 
-        attr_reader :variable, :compare_key, :value, :path
+        attr_reader :variable, :compare_key, :type, :compare_predicate, :path
 
         def initialize(_workflow, _name, payload)
           super
 
-          @variable = parse_path("Variable", payload)
+          @variable = parse_path("Variable")
           parse_compare_key
-          @value = path ? parse_path(compare_key, payload) : payload[compare_key]
         end
 
         def true?(context, input)
@@ -108,26 +112,74 @@ module Floe
         # rubocop:enable Naming/PredicateName
         # rubocop:enable Style/OptionalBooleanParameter
 
+        # parse the compare key at initialization time
         def parse_compare_key
-          @compare_key = payload.keys.detect { |key| key.match?(/^(#{COMPARE_KEYS.join("|")})/) }
+          payload.each_key do |key|
+            # e.g. (String)(GreaterThan)(Path)
+            if (match_values = OPERATION.match(key))
+              @compare_key = key
+              @type, _operator, @path = match_values.captures
+              @compare_predicate = parse_predicate(type)
+              break
+            end
+            # e.g. (Is)(String)
+            if TYPE_CHECK.match?(key)
+              @compare_key = key
+              # type: nil means no runtime type checking.
+              @type = @path = nil
+              @compare_predicate = parse_predicate("Boolean")
+              break
+            end
+          end
           parser_error!("requires a compare key") unless compare_key
+        end
 
-          @path = compare_key.end_with?("Path")
+        # parse predicate at initilization time
+        # @return the right predicate attached to the compare key
+        def parse_predicate(data_type)
+          path ? parse_path(compare_key) : parse_field(compare_key, data_type)
         end
 
+        # @return right hand predicate - input path or static payload value)
         def compare_value(context, input)
-          path ? value.value(context, input) : value
+          path ? fetch_path(compare_key, compare_predicate, context, input) : compare_predicate
         end
 
+        # feth the variable value at runtime
+        # @return variable value (left hand side )
         def variable_value(context, input)
-          variable.value(context, input)
+          fetch_path("Variable", variable, context, input)
         end
 
-        def parse_path(field_name, payload)
+        # parse path at initilization time
+        # helper method to parse a path from the payload
+        def parse_path(field_name)
           value = payload[field_name]
           missing_field_error!(field_name) unless value
           wrap_parser_error(field_name, value) { Path.new(value) }
         end
+
+        # parse predicate field at initialization time
+        def parse_field(field_name, data_type)
+          value = payload[field_name]
+          return value if correct_type?(value, data_type)
+
+          invalid_field_error!(field_name, value, "required to be a #{data_type}")
+        end
+
+        # fetch a path at runtime
+        def fetch_path(field_name, field_path, context, input)
+          value = field_path.value(context, input)
+          return value if type.nil? || correct_type?(value, type)
+
+          runtime_field_error!(field_name, field_path.to_s, "required to point to a #{type}")
+        end
+
+        # if we have runtime checking, check against that type
+        #   otherwise assume checking a TYPE_CHECK predicate and check against Boolean
+        def correct_type?(value, data_type)
+          send("is_#{data_type.downcase}?".to_sym, value)
+        end
       end
     end
   end

data/lib/floe/workflow/item_processor.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ItemProcessor < Floe::WorkflowBase
+      attr_reader :processor_config
+
+      def initialize(payload, name = nil)
+        super
+        @processor_config = payload.fetch("ProcessorConfig", "INLINE")
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/child_workflow_mixin.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      module ChildWorkflowMixin
+        def run_nonblock!(context)
+          start(context) unless context.state_started?
+
+          step_nonblock!(context) while running?(context)
+          return Errno::EAGAIN unless ready?(context)
+
+          finish(context) if ended?(context)
+        end
+
+        def finish(context)
+          if success?(context)
+            result = each_child_context(context).map(&:output)
+            context.output = process_output(context, result)
+          else
+            error = parse_error(context)
+            retry_state!(context, error) || catch_error!(context, error) || fail_workflow!(context, error)
+          end
+
+          super
+        end
+
+        def ready?(context)
+          !context.state_started? || each_child_workflow(context).any? { |wf, ctx| wf.step_nonblock_ready?(ctx) }
+        end
+
+        def wait_until(context)
+          each_child_workflow(context).filter_map { |wf, ctx| wf.wait_until(ctx) }.min
+        end
+
+        def waiting?(context)
+          each_child_workflow(context).any? { |wf, ctx| wf.waiting?(ctx) }
+        end
+
+        def running?(context)
+          !ended?(context)
+        end
+
+        def ended?(context)
+          each_child_context(context).all?(&:ended?)
+        end
+
+        def success?(context)
+          each_child_context(context).none?(&:failed?)
+        end
+
+        def each_child_context(context)
+          context.state[child_context_key].map { |ctx| Context.new(ctx) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/choice.rb

@@ -9,10 +9,9 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          validate_state!(workflow)
-
-          @choices = payload["Choices"].map.with_index { |choice, i| ChoiceRule.build(workflow, name + ["Choices", i.to_s], choice) }
+          @choices = payload["Choices"]&.map&.with_index { |choice, i| ChoiceRule.build(workflow, name + ["Choices", i.to_s], choice) }
           @default = payload["Default"]
+          validate_state!(workflow)
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
@@ -45,12 +44,12 @@ module Floe
         end
 
         def validate_state_choices!
-          missing_field_error!("Choices") unless payload.key?("Choices")
-          invalid_field_error!("Choices", nil, "must be a non-empty array") unless payload["Choices"].kind_of?(Array) && !payload["Choices"].empty?
+          missing_field_error!("Choices") if @choices.nil?
+          invalid_field_error!("Choices", nil, "must be a non-empty array") unless @choices.kind_of?(Array) && !@choices.empty?
         end
 
         def validate_state_default!(workflow)
-          invalid_field_error!("Default", payload["Default"], "is not found in \"States\"") if payload["Default"] && !workflow_state?(payload["Default"], workflow)
+          invalid_field_error!("Default", @default, "is not found in \"States\"") if @default && !workflow_state?(@default, workflow)
         end
       end
     end

data/lib/floe/workflow/states/map.rb

@@ -4,9 +4,123 @@ module Floe
   class Workflow
     module States
       class Map < Floe::Workflow::State
-        def initialize(*)
+        include ChildWorkflowMixin
+        include InputOutputMixin
+        include NonTerminalMixin
+        include RetryCatchMixin
+
+        attr_reader :end, :next, :parameters, :input_path, :output_path, :result_path,
+                    :result_selector, :retry, :catch, :item_processor, :items_path,
+                    :item_reader, :item_selector, :item_batcher, :result_writer,
+                    :max_concurrency, :tolerated_failure_percentage, :tolerated_failure_count
+
+        def initialize(workflow, name, payload)
+          super
+
+          missing_field_error!("InputProcessor") if payload["ItemProcessor"].nil?
+
+          @next            = payload["Next"]
+          @end             = !!payload["End"]
+          @parameters      = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
+          @input_path      = Path.new(payload.fetch("InputPath", "$"))
+          @output_path     = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path     = ReferencePath.new(payload.fetch("ResultPath", "$"))
+          @result_selector = PayloadTemplate.new(payload["ResultSelector"]) if payload["ResultSelector"]
+          @retry           = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
+          @catch           = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @item_processor  = ItemProcessor.new(payload["ItemProcessor"], name)
+          @items_path      = ReferencePath.new(payload.fetch("ItemsPath", "$"))
+          @item_reader     = payload["ItemReader"]
+          @item_selector   = payload["ItemSelector"]
+          @item_batcher    = payload["ItemBatcher"]
+          @result_writer   = payload["ResultWriter"]
+          @max_concurrency = payload["MaxConcurrency"]&.to_i
+          @tolerated_failure_percentage = payload["ToleratedFailurePercentage"]&.to_i
+          @tolerated_failure_count      = payload["ToleratedFailureCount"]&.to_i
+
+          validate_state!(workflow)
+        end
+
+        def process_input(context)
+          input = super
+          items_path.value(context, input)
+        end
+
+        def start(context)
           super
-          raise NotImplementedError
+
+          input = process_input(context)
+
+          context.state["ItemProcessorContext"] = input.map { |item| Context.new({"Execution" => {"Id" => context.execution["Id"]}}, :input => item.to_json).to_h }
+        end
+
+        def end?
+          @end
+        end
+
+        def success?(context)
+          contexts   = each_child_context(context)
+          num_failed = contexts.count(&:failed?)
+          total      = contexts.count
+
+          return true if num_failed.zero? || total.zero?
+          return false if tolerated_failure_count.nil? && tolerated_failure_percentage.nil?
+
+          # Some have failed, check the tolerated_failure thresholds to see if
+          # we should fail the whole state.
+          #
+          # If either ToleratedFailureCount or ToleratedFailurePercentage are breached
+          # then the whole state is considered failed.
+          count_tolerated = tolerated_failure_count.nil?      || num_failed < tolerated_failure_count
+          pct_tolerated   = tolerated_failure_percentage.nil? || tolerated_failure_percentage == 100 ||
+                            ((100 * num_failed / total.to_f) < tolerated_failure_percentage)
+
+          count_tolerated && pct_tolerated
+        end
+
+        private
+
+        def step_nonblock!(context)
+          each_child_context(context).each do |ctx|
+            # If this iteration isn't already running and we can't start any more
+            next if !ctx.started? && concurrency_exceeded?(context)
+
+            item_processor.run_nonblock(ctx) if item_processor.step_nonblock_ready?(ctx)
+          end
+        end
+
+        def each_child_workflow(context)
+          each_child_context(context).map do |ctx|
+            [item_processor, Context.new(ctx)]
+          end
+        end
+
+        def concurrency_exceeded?(context)
+          max_concurrency && num_running(context) >= max_concurrency
+        end
+
+        def num_running(context)
+          each_child_context(context).count(&:running?)
+        end
+
+        def parse_error(context)
+          # If ToleratedFailureCount or ToleratedFailurePercentage is present
+          # then use States.ExceedToleratedFailureThreshold otherwise
+          # take the error from the first failed state
+          if tolerated_failure_count || tolerated_failure_percentage
+            {"Error" => "States.ExceedToleratedFailureThreshold"}
+          else
+            each_child_context(context).detect(&:failed?)&.output || {"Error" => "States.Error"}
+          end
+        end
+
+        def child_context_key
+          "ItemProcessorContext"
+        end
+
+        def validate_state!(workflow)
+          validate_state_next!(workflow)
+          invalid_field_error!("MaxConcurrency", @max_concurrency, "must be greater than 0") if @max_concurrency && @max_concurrency <= 0
         end
       end
     end

data/lib/floe/workflow/states/parallel.rb

@@ -4,9 +4,72 @@ module Floe
   class Workflow
     module States
       class Parallel < Floe::Workflow::State
-        def initialize(*)
+        include ChildWorkflowMixin
+        include InputOutputMixin
+        include NonTerminalMixin
+        include RetryCatchMixin
+
+        attr_reader :end, :next, :parameters, :input_path, :output_path, :result_path,
+                    :result_selector, :retry, :catch, :branches
+
+        def initialize(workflow, name, payload)
+          super
+
+          missing_field_error!("Branches") if payload["Branches"].nil?
+
+          @next            = payload["Next"]
+          @end             = !!payload["End"]
+          @parameters      = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
+          @input_path      = Path.new(payload.fetch("InputPath", "$"))
+          @output_path     = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path     = ReferencePath.new(payload.fetch("ResultPath", "$"))
+          @result_selector = PayloadTemplate.new(payload["ResultSelector"]) if payload["ResultSelector"]
+          @retry           = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
+          @catch           = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @branches        = payload["Branches"].map { |branch| Branch.new(branch) }
+
+          validate_state!(workflow)
+        end
+
+        def start(context)
           super
-          raise NotImplementedError
+
+          input = process_input(context)
+
+          context.state["BranchContext"] = branches.map { |_branch| Context.new({"Execution" => {"Id" => context.execution["Id"]}}, :input => input.to_json).to_h }
+        end
+
+        def end?
+          @end
+        end
+
+        private
+
+        def step_nonblock!(context)
+          each_child_workflow(context).each do |wf, ctx|
+            wf.run_nonblock(ctx) if wf.step_nonblock_ready?(ctx)
+          end
+        end
+
+        def each_child_workflow(context)
+          branches.filter_map.with_index do |branch, i|
+            ctx = context.state.dig("BranchContext", i)
+            next if ctx.nil?
+
+            [branch, Context.new(ctx)]
+          end
+        end
+
+        def parse_error(context)
+          each_child_context(context).detect(&:failed?)&.output || {"Error" => "States.Error"}
+        end
+
+        def child_context_key
+          "BranchContext"
+        end
+
+        def validate_state!(workflow)
+          validate_state_next!(workflow)
         end
       end
     end

data/lib/floe/workflow/states/retry_catch_mixin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      module RetryCatchMixin
+        def find_retrier(error)
+          self.retry.detect { |r| r.match_error?(error) }
+        end
+
+        def find_catcher(error)
+          self.catch.detect { |c| c.match_error?(error) }
+        end
+
+        def retry_state!(context, error)
+          retrier = find_retrier(error["Error"]) if error
+          return if retrier.nil?
+
+          # If a different retrier is hit reset the context
+          if !context["State"].key?("RetryCount") || context["State"]["Retrier"] != retrier.error_equals
+            context["State"]["RetryCount"] = 0
+            context["State"]["Retrier"]    = retrier.error_equals
+          end
+
+          context["State"]["RetryCount"] += 1
+
+          return if context["State"]["RetryCount"] > retrier.max_attempts
+
+          wait_until!(context, :seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
+          context.next_state = context.state_name
+          context.output     = error
+          logger.info("Running state: [#{long_name}] with input [#{context.json_input}] got error[#{context.json_output}]...Retry - delay: #{wait_until(context)}")
+          true
+        end
+
+        def catch_error!(context, error)
+          catcher = find_catcher(error["Error"]) if error
+          return if catcher.nil?
+
+          context.next_state = catcher.next
+          context.output     = catcher.result_path.set(context.input, error)
+          logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...CatchError - next state: [#{context.next_state}] output: [#{context.json_output}]")
+
+          true
+        end
+
+        def fail_workflow!(context, error)
+          # next_state is nil, and will be set to nil again in super
+          # keeping in here for completeness
+          context.next_state = nil
+          context.output = error
+          logger.error("Running state: [#{long_name}] with input [#{context.json_input}]...Complete workflow - output: [#{context.json_output}]")
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/task.rb

@@ -6,6 +6,7 @@ module Floe
       class Task < Floe::Workflow::State
         include InputOutputMixin
         include NonTerminalMixin
+        include RetryCatchMixin
 
         attr_reader :credentials, :end, :heartbeat_seconds, :next, :parameters,
                     :result_selector, :resource, :timeout_seconds, :retry, :catch,
@@ -82,54 +83,6 @@ module Floe
           runner.success?(context.state["RunnerContext"])
         end
 
-        def find_retrier(error)
-          self.retry.detect { |r| r.match_error?(error) }
-        end
-
-        def find_catcher(error)
-          self.catch.detect { |c| c.match_error?(error) }
-        end
-
-        def retry_state!(context, error)
-          retrier = find_retrier(error["Error"]) if error
-          return if retrier.nil?
-
-          # If a different retrier is hit reset the context
-          if !context["State"].key?("RetryCount") || context["State"]["Retrier"] != retrier.error_equals
-            context["State"]["RetryCount"] = 0
-            context["State"]["Retrier"]    = retrier.error_equals
-          end
-
-          context["State"]["RetryCount"] += 1
-
-          return if context["State"]["RetryCount"] > retrier.max_attempts
-
-          wait_until!(context, :seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
-          context.next_state = context.state_name
-          context.output     = error
-          logger.info("Running state: [#{long_name}] with input [#{context.json_input}] got error[#{context.json_output}]...Retry - delay: #{wait_until(context)}")
-          true
-        end
-
-        def catch_error!(context, error)
-          catcher = find_catcher(error["Error"]) if error
-          return if catcher.nil?
-
-          context.next_state = catcher.next
-          context.output     = catcher.result_path.set(context.input, error)
-          logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...CatchError - next state: [#{context.next_state}] output: [#{context.json_output}]")
-
-          true
-        end
-
-        def fail_workflow!(context, error)
-          # next_state is nil, and will be set to nil again in super
-          # keeping in here for completeness
-          context.next_state = nil
-          context.output = error
-          logger.error("Running state: [#{long_name}] with input [#{context.json_input}]...Complete workflow - output: [#{context.json_output}]")
-        end
-
         def parse_error(output)
           return if output.nil?
           return output if output.kind_of?(Hash)

data/lib/floe/workflow.rb

@@ -4,9 +4,8 @@ require "securerandom"
 require "json"
 
 module Floe
-  class Workflow
+  class Workflow < Floe::WorkflowBase
     include Logging
-    include ValidationMixin
 
     class << self
       def load(path_or_io, context = nil, credentials = {}, name = nil)
@@ -19,7 +18,7 @@ module Floe
 
       def wait(workflows, timeout: nil, &block)
         workflows = [workflows] if workflows.kind_of?(self)
-        logger.info("checking #{workflows.count} workflows...")
+        logger.info("Checking #{workflows.count} workflows...")
 
         run_until   = Time.now.utc + timeout if timeout.to_i > 0
         ready       = []
@@ -66,29 +65,21 @@ module Floe
             event, data = queue.pop
             break if event.nil?
 
-            _execution_id, runner_context = data.values_at("execution_id", "runner_context")
-
-            # If the event is for one of our workflows set the updated runner_context
-            workflows.each do |workflow|
-              next unless workflow.context.state.dig("RunnerContext", "container_ref") == runner_context["container_ref"]
-
-              workflow.context.state["RunnerContext"] = runner_context
-            end
-
-            break if queue.empty?
+            # break out of the loop if the event is for one of our workflows
+            break if queue.empty? || workflows.detect { |wf| wf.execution_id == data["execution_id"] }
           end
         ensure
           sleep_thread&.kill
         end
 
-        logger.info("checking #{workflows.count} workflows...Complete - #{ready.count} ready")
+        logger.info("Checking #{workflows.count} workflows...Complete - #{ready.count} ready")
         ready
       ensure
         wait_thread&.kill
       end
     end
 
-    attr_reader :context, :payload, :states, :states_by_name, :start_at, :name, :comment
+    attr_reader :comment, :context
 
     def initialize(payload, context = nil, credentials = nil, name = nil)
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
@@ -99,20 +90,10 @@ module Floe
       # caller should really put credentials into context and not pass that variable
       context.credentials = credentials if credentials
 
-      # NOTE: this is a string, and states use an array
-      @name        = name || "State Machine"
-      @payload     = payload
-      @context     = context
-      @comment     = payload["Comment"]
-      @start_at    = payload["StartAt"]
-
-      # NOTE: Everywhere else we include our name (i.e.: parent name) when building the child name.
-      #       When creating the states, we are dropping our name (i.e.: the workflow name)
-      @states      = payload["States"].to_a.map { |state_name, state| State.build!(self, ["States", state_name], state) }
+      @context = context
+      @comment = payload["Comment"]
 
-      validate_workflow
-
-      @states_by_name = @states.each_with_object({}) { |state, result| result[state.short_name] = state }
+      super(payload, name)
     rescue Floe::Error
       raise
     rescue => err
@@ -185,7 +166,7 @@ module Floe
 
     # NOTE: Expecting the context to be initialized (via start_workflow) before this
     def current_state
-      @states_by_name[context.state_name]
+      states_by_name[context.state_name]
     end
 
     # backwards compatibility. Caller should access directly from context
@@ -193,14 +174,12 @@ module Floe
       @context.credentials
     end
 
-    private
-
-    def validate_workflow
-      missing_field_error!("States") if @states.empty?
-      missing_field_error!("StartAt") if @start_at.nil?
-      invalid_field_error!("StartAt", @start_at, "is not found in \"States\"") unless workflow_state?(@start_at, self)
+    def execution_id
+      @context.execution["Id"]
     end
 
+    private
+
     def step!
       next_state = {"Name" => context.next_state, "Guid" => SecureRandom.uuid, "PreviousStateGuid" => context.state["Guid"]}
 

data/lib/floe/workflow_base.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Floe
+  class WorkflowBase
+    include ValidationMixin
+
+    attr_reader :name, :payload, :start_at, :states, :states_by_name
+
+    def initialize(payload, name = nil)
+      # NOTE: this is a string, and states use an array
+      @name     = name || "State Machine"
+      @payload  = payload
+      @start_at = payload["StartAt"]
+
+      # NOTE: Everywhere else we include our name (i.e.: parent name) when building the child name.
+      #       When creating the states, we are dropping our name (i.e.: the workflow name)
+      @states         = payload["States"].to_a.map { |state_name, state| Floe::Workflow::State.build!(self, ["States", state_name], state) }
+      @states_by_name = @states.to_h { |state| [state.short_name, state] }
+
+      validate_workflow!
+    end
+
+    def run(context)
+      run_nonblock(context) until context.ended?
+    end
+
+    def run_nonblock(context)
+      start_workflow(context)
+      loop while step_nonblock(context) == 0 && !context.ended?
+      self
+    end
+
+    def step_nonblock(context)
+      return Errno::EPERM if context.ended?
+
+      result = current_state(context).run_nonblock!(context)
+      return result if result != 0
+
+      context.state_history << context.state
+      context.next_state ? step!(context) : end_workflow!(context)
+
+      result
+    end
+
+    def step_nonblock_ready?(context)
+      !context.started? || current_state(context).ready?(context)
+    end
+
+    def waiting?(context)
+      current_state(context)&.waiting?(context)
+    end
+
+    def wait_until(context)
+      current_state(context)&.wait_until(context)
+    end
+
+    def start_workflow(context)
+      return if context.state_name
+
+      context.state["Name"]  = start_at
+      context.state["Input"] = context.execution["Input"].dup
+
+      context.execution["StartTime"] = Time.now.utc.iso8601
+
+      self
+    end
+
+    def current_state(context)
+      states_by_name[context.state_name]
+    end
+
+    def end?(context)
+      context.ended?
+    end
+
+    def output(context)
+      context.output.to_json if end?(context)
+    end
+
+    private
+
+    def step!(context)
+      next_state = {"Name" => context.next_state}
+
+      # if rerunning due to an error (and we are using Retry)
+      if context.state_name == context.next_state && context.failed? && context.state.key?("Retrier")
+        next_state.merge!(context.state.slice("RetryCount", "Input", "Retrier"))
+      else
+        next_state["Input"] = context.output
+      end
+
+      context.state = next_state
+    end
+
+    # Avoiding State#running? because that is potentially expensive.
+    # State#run_nonblock! already called running? via State#ready? and
+    # called State#finished -- which is what Context#state_finished? is detecting
+    def end_workflow!(context)
+      context.execution["EndTime"] = context.state["FinishedTime"]
+    end
+
+    def validate_workflow!
+      missing_field_error!("States") if @states.empty?
+      missing_field_error!("StartAt") if @start_at.nil?
+      invalid_field_error!("StartAt", @start_at, "is not found in \"States\"") unless workflow_state?(@start_at, self)
+    end
+  end
+end

data/lib/floe.rb

@@ -8,8 +8,11 @@ require_relative "floe/logging"
 require_relative "floe/runner"
 
 require_relative "floe/validation_mixin"
+require_relative "floe/workflow_base"
 require_relative "floe/workflow"
+# mixins used by workflow components
 require_relative "floe/workflow/error_matcher_mixin"
+require_relative "floe/workflow/branch"
 require_relative "floe/workflow/catcher"
 require_relative "floe/workflow/choice_rule"
 require_relative "floe/workflow/choice_rule/not"
@@ -17,6 +20,7 @@ require_relative "floe/workflow/choice_rule/or"
 require_relative "floe/workflow/choice_rule/and"
 require_relative "floe/workflow/choice_rule/data"
 require_relative "floe/workflow/context"
+require_relative "floe/workflow/item_processor"
 require_relative "floe/workflow/intrinsic_function"
 require_relative "floe/workflow/intrinsic_function/parser"
 require_relative "floe/workflow/intrinsic_function/transformer"
@@ -25,11 +29,14 @@ require_relative "floe/workflow/payload_template"
 require_relative "floe/workflow/reference_path"
 require_relative "floe/workflow/retrier"
 require_relative "floe/workflow/state"
+# mixins used by states
+require_relative "floe/workflow/states/child_workflow_mixin"
+require_relative "floe/workflow/states/input_output_mixin"
+require_relative "floe/workflow/states/non_terminal_mixin"
+require_relative "floe/workflow/states/retry_catch_mixin"
 require_relative "floe/workflow/states/choice"
 require_relative "floe/workflow/states/fail"
-require_relative "floe/workflow/states/input_output_mixin"
 require_relative "floe/workflow/states/map"
-require_relative "floe/workflow/states/non_terminal_mixin"
 require_relative "floe/workflow/states/parallel"
 require_relative "floe/workflow/states/pass"
 require_relative "floe/workflow/states/succeed"

data/tools/step_functions

@@ -0,0 +1,110 @@
+#!/usr/bin/env ruby
+
+require "bundler/inline"
+gemfile do
+  source "https://rubygems.org"
+  gem "optimist"
+  gem "colorize"
+end
+require "pp"
+
+SUB_COMMANDS = {
+  "execute"   => "Execute an .asl file through the stepfunctions simulator.",
+  "intrinsic" => "Execute an intrinsic function or JSONPath standalone."
+}.freeze
+Optimist.options do
+  banner "Run the aws stepfunctions simulator."
+  banner ""
+  banner "Notes:"
+  banner "  This tool requires the stepfunctions simulator to be installed locally and running."
+  banner "  Installation instructions can be found at https://docs.aws.amazon.com/step-functions/latest/dg/sfn-local.html."
+  banner ""
+  banner "Commands:"
+  SUB_COMMANDS.each { |k, v| banner "  #{k.ljust(14)}#{v}" }
+  banner ""
+  banner "  For more help with a specific command use #{$PROGRAM_NAME} <command> --help"
+  banner ""
+  banner "Global Options:"
+  stop_on SUB_COMMANDS.keys
+end
+cmd = ARGV.shift
+Optimist.educate if cmd.nil?
+Optimist.die "unknown subcommand #{cmd.inspect}" unless SUB_COMMANDS.include?(cmd)
+
+def aws_stepfunctions(args)
+  cmd = "aws stepfunctions --endpoint-url http://localhost:8083 #{args}"
+  puts "** #{cmd}".light_black if ENV["DEBUG"]
+  output = `#{cmd}`.chomp
+  output = output.empty? ? {} : JSON.parse(output)
+  puts output.pretty_inspect.light_black if ENV["DEBUG"]
+  output
+rescue JSON::ParserError => err
+  warn "ERROR: #{err}".light_red if ENV["DEBUG"]
+  {}
+end
+
+def execute_stepfunction(definition, input)
+  require "json"
+  require "shellwords"
+
+  begin
+    state_machine_arn = aws_stepfunctions("create-state-machine --definition #{Shellwords.escape(definition)} --name 'StateMachine' --role-arn 'arn:aws:iam::012345678901:role/DummyRole'")["stateMachineArn"]
+    exit 1 if state_machine_arn.nil?
+
+    input = input ? "--input #{Shellwords.escape(input)}" : ""
+    execution_arn = aws_stepfunctions("start-execution --state-machine-arn #{state_machine_arn} #{input}")["executionArn"]
+    exit 1 if execution_arn.nil?
+
+    status, output = aws_stepfunctions("describe-execution --execution-arn #{execution_arn}").values_at("status", "output")
+    if status == "FAILED"
+      warn "ERROR: Execution failed. See simulator for reason.".light_red
+      exit 1
+    end
+  ensure
+    aws_stepfunctions("stop-execution --execution-arn #{execution_arn}") if execution_arn
+    aws_stepfunctions("delete-state-machine --state-machine-arn #{state_machine_arn}") if state_machine_arn
+  end
+
+  puts output if output
+end
+
+def execute
+  opts = Optimist.options do
+    banner SUB_COMMANDS["execute"]
+    banner ""
+
+    opt :file,  "The .asl file to execute", :default => "definition.asl"
+    opt :input, "Input to the execution", :type => :string
+  end
+
+  definition = File.read(opts[:file]).chomp
+  execute_stepfunction(definition, opts[:input])
+end
+
+def intrinsic
+  opts = Optimist.options do
+    banner SUB_COMMANDS["intrinsic"]
+    banner ""
+
+    opt :function, "The intrinsic function or JSONPath to run", :type => :string, :required => true
+    opt :input, "Input to the execution", :type => :string
+  end
+
+  require "json"
+
+  definition = {
+    "StartAt" => "ExecState",
+    "States"  => {
+      "ExecState" => {
+        "Type"       => "Pass",
+        "Parameters" => {"data.$" => opts[:function]},
+        "OutputPath" => "$.data",
+        "End"        => true
+      }
+    }
+  }.to_json
+
+  execute_stepfunction(definition, opts[:input])
+end
+
+send(cmd)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-08-20 00:00:00.000000000 Z
+date: 2024-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -182,6 +182,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/map.asl
+- examples/parallel.asl
 - examples/set-credential.asl
 - examples/workflow.asl
 - exe/floe
@@ -199,6 +201,7 @@ files:
 - lib/floe/validation_mixin.rb
 - lib/floe/version.rb
 - lib/floe/workflow.rb
+- lib/floe/workflow/branch.rb
 - lib/floe/workflow/catcher.rb
 - lib/floe/workflow/choice_rule.rb
 - lib/floe/workflow/choice_rule/and.rb
@@ -210,11 +213,13 @@ files:
 - lib/floe/workflow/intrinsic_function.rb
 - lib/floe/workflow/intrinsic_function/parser.rb
 - lib/floe/workflow/intrinsic_function/transformer.rb
+- lib/floe/workflow/item_processor.rb
 - lib/floe/workflow/path.rb
 - lib/floe/workflow/payload_template.rb
 - lib/floe/workflow/reference_path.rb
 - lib/floe/workflow/retrier.rb
 - lib/floe/workflow/state.rb
+- lib/floe/workflow/states/child_workflow_mixin.rb
 - lib/floe/workflow/states/choice.rb
 - lib/floe/workflow/states/fail.rb
 - lib/floe/workflow/states/input_output_mixin.rb
@@ -222,11 +227,13 @@ files:
 - lib/floe/workflow/states/non_terminal_mixin.rb
 - lib/floe/workflow/states/parallel.rb
 - lib/floe/workflow/states/pass.rb
+- lib/floe/workflow/states/retry_catch_mixin.rb
 - lib/floe/workflow/states/succeed.rb
 - lib/floe/workflow/states/task.rb
 - lib/floe/workflow/states/wait.rb
+- lib/floe/workflow_base.rb
 - renovate.json
-- sig/floe.rbs/floe.rbs
+- tools/step_functions
 homepage: https://github.com/ManageIQ/floe
 licenses:
 - Apache-2.0

data/sig/floe.rbs/floe.rbs

@@ -1,4 +0,0 @@
-module Floe
-  VERSION: String
-  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
-end