floe 0.14.0 → 0.15.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0e0b86c08d322e2ebc8e115e5193d0493f8257c831f7377b18c32d4284ae6f2
-  data.tar.gz: 1bf5ed62abafe2e1af6025dbaf47d9faa3b1c93ffec8fd88fbeacd4445efc795
+  metadata.gz: 6f21cf2fcc6a7fa9f7536e0cca328d101a867c389f53f768e4746b6e69f643a5
+  data.tar.gz: f3d82401842f66504afdeaa4b051db01d5fcb9e79b704af368712d68b4950251
 SHA512:
-  metadata.gz: 34f64555f2472c121fcea9dda8baac7fbe853cdbc5e949c831d7b7f070878d3ef2f86ed90d907b5a7033539fcad6aeea64a4dee6475a752d7f89e4f679a4b269
-  data.tar.gz: 378c5bcd562d4bbeb7e3eb13eb55b1d5d8e7aed0ea4749c2b65f951cf3be0e5f13d3e288cdbf63b4a5729e98c5010656d6dca3db60ffbf8463988008980edd83
+  metadata.gz: 3e73dbdd868ec57b8138a9e4b0b529c0f2c44cdb21d43c0ae29ca9094512f4044cc621a38e4ac217bf6b17e13f1ef3a361cf5917023eab87d30c7b4cc90eccfc
+  data.tar.gz: e174fabde62d6348f8517f7c2d019600b34612f5b8d822eb83c2c5579f5e224596b6edaec383bfe9cce75c3260e50a4a0b744764119cd96e2a973f92b49b7d5a

data/.codeclimate.yml

@@ -1,3 +1,4 @@
+version: '2'
 prepare:
   fetch:
   - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/.rubocop_base.yml
@@ -8,9 +9,28 @@ prepare:
     path: styles/base.yml
   - url: https://raw.githubusercontent.com/ManageIQ/manageiq-style/master/styles/cc_base.yml
     path: styles/cc_base.yml
+checks:
+  argument-count:
+    enabled: false
+  complex-logic:
+    enabled: false
+  file-lines:
+    enabled: false
+  method-complexity:
+    config:
+      threshold: 11
+  method-count:
+    enabled: false
+  method-lines:
+    enabled: false
+  nested-control-flow:
+    enabled: false
+  return-statements:
+    enabled: false
 plugins:
   rubocop:
     enabled: true
     config: ".rubocop_cc.yml"
     channel: rubocop-1-56-3
-version: '2'
+exclude_patterns:
+- spec/

data/CHANGELOG.md

@@ -4,6 +4,29 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.15.1] - 2024-11-21
+### Fixed
+- Fix Map/Parallel States checking container_runner#status! of finished states ([#296](https://github.com/ManageIQ/floe/pull/296))
+- Fix child workflow mixin tight loop ([#297](https://github.com/ManageIQ/floe/pull/297))
+
+## [0.15.0] - 2024-10-28
+### Added
+- Add WorkflowBase base class for Workflow ([#279](https://github.com/ManageIQ/floe/pull/279))
+- Add tool for using the aws stepfunctions simulator ([#244](https://github.com/ManageIQ/floe/pull/244))
+- Implement Map state ([#184](https://github.com/ManageIQ/floe/pull/184))
+- Add Map State Tolerated Failure ([#282](https://github.com/ManageIQ/floe/pull/282))
+- Run Map iterations in parallel up to MaxConcurrency ([#283](https://github.com/ManageIQ/floe/pull/283))
+- Implement Parallel State ([#291](https://github.com/ManageIQ/floe/pull/291))
+
+### Changed
+- More granular compare_key and determine path at initialization time ([#274](https://github.com/ManageIQ/floe/pull/274))
+- For Choice validation, use instance variables and not payload ([#277](https://github.com/ManageIQ/floe/pull/277))
+- Return ExceedToleratedFailureThreshold if ToleratedFailureCount/Percentage is present ([#285](https://github.com/ManageIQ/floe/pull/285))
+
+### Fixed
+- Fix case on log messages ([#280](https://github.com/ManageIQ/floe/pull/280))
+- Handle either ToleratedFailureCount or ToleratedFailurePercentage ([#284](https://github.com/ManageIQ/floe/pull/284))
+
 ## [0.14.0] - 2024-08-20
 ### Added
 - Implement "IsNumeric": false ([#266](https://github.com/ManageIQ/floe/pull/266))
@@ -249,7 +272,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.14.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.15.1...HEAD
+[0.15.1]: https://github.com/ManageIQ/floe/compare/v0.15.0...v0.15.1
+[0.15.0]: https://github.com/ManageIQ/floe/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/ManageIQ/floe/compare/v0.13.1...v0.14.0
 [0.13.1]: https://github.com/ManageIQ/floe/compare/v0.13.0...v0.13.1
 [0.13.0]: https://github.com/ManageIQ/floe/compare/v0.12.0...v0.13.0

data/README.md

@@ -197,6 +197,14 @@ Options supported by the kubernetes docker runner are:
 * `ca_file` - Path to a certificate-authority file for the kubernetes API, only valid if server and token are passed.  If present `/run/secrets/kubernetes.io/serviceaccount/ca.crt` will be used
 * `verify_ssl` - Controls if the kubernetes API certificate-authority should be verified, defaults to "true", only vaild if server and token are passed
 
+## Features Not Yet Supported
+
+The following are not yet supported:
+- Map State Fields:
+  - ItemReader
+  - ItemSelector/ItemBatcher
+  - ResultWriter
+
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. You can also run `bin/console` for an interactive prompt that will allow you to experiment.

data/examples/map.asl

@@ -0,0 +1,46 @@
+{
+  "Comment": "Using Map state in Inline mode",
+  "StartAt": "Pass",
+  "States": {
+    "Pass": {
+      "Type": "Pass",
+      "Next": "Map demo",
+      "Result": {
+        "foo": "bar",
+        "colors": [
+          "red",
+          "green",
+          "blue",
+          "yellow",
+          "white"
+        ]
+      }
+    },
+    "Map demo": {
+      "Type": "Map",
+      "ItemsPath": "$.colors",
+      "MaxConcurrency": 2,
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "INLINE"
+        },
+        "StartAt": "Generate UUID",
+        "States": {
+          "Generate UUID": {
+            "Type": "Pass",
+            "Next": "Sleep",
+            "Parameters": {
+              "uuid.$": "States.UUID()"
+            }
+          },
+          "Sleep": {
+            "Type": "Task",
+            "Resource": "docker://docker.io/agrare/sleep:latest",
+            "End": true
+          }
+        }
+      },
+      "End": true
+    }
+  }
+}

data/examples/parallel.asl

@@ -0,0 +1,32 @@
+{
+  "Comment": "Parallel Example.",
+  "StartAt": "FunWithMath",
+  "States": {
+    "FunWithMath": {
+      "Type": "Parallel",
+      "End": true,
+      "Branches": [
+        {
+          "StartAt": "Add",
+          "States": {
+            "Add": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        },
+        {
+          "StartAt": "Subtract",
+          "States": {
+            "Subtract": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        }
+      ]
+    }
+  }
+}

data/lib/floe/cli.rb

@@ -1,9 +1,12 @@
+require "floe"
+require "floe/container_runner"
+
 module Floe
   class CLI
+    include Logging
+
     def initialize
       require "optimist"
-      require "floe"
-      require "floe/container_runner"
       require "logger"
 
       Floe.logger = Logger.new($stdout)
@@ -13,25 +16,29 @@ module Floe
     def run(args = ARGV)
       workflows_inputs, opts = parse_options!(args)
 
-      credentials =
-        if opts[:credentials_given]
-          opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
-        elsif opts[:credentials_file_given]
-          File.read(opts[:credentials_file])
-        end
+      credentials = create_credentials(opts)
 
       workflows =
         workflows_inputs.each_slice(2).map do |workflow, input|
-          context = Floe::Workflow::Context.new(opts[:context], :input => input, :credentials => credentials)
-          Floe::Workflow.load(workflow, context)
+          create_workflow(workflow, opts[:context], input, credentials)
         end
 
-      Floe::Workflow.wait(workflows, &:run_nonblock)
+      output_streams = create_loggers(workflows, opts[:segment_output])
+
+      logger.info("Checking #{workflows.count} workflows...")
+      ready = Floe::Workflow.wait(workflows, &:run_nonblock)
+      logger.info("Checking #{workflows.count} workflows...Complete - #{ready.count} ready")
 
       # Display status
       workflows.each do |workflow|
-        puts "", "#{workflow.name}#{" (#{workflow.status})" unless workflow.context.success?}", "===" if workflows.size > 1
-        puts workflow.output
+        if workflows.size > 1
+          logger.info("")
+          logger.info("#{workflow.name}#{" (#{workflow.status})" unless workflow.context.success?}")
+          logger.info("===")
+        end
+
+        logger.info(output_streams[workflow].string) if output_streams[workflow]
+        logger.info(workflow.output)
       end
 
       workflows.all? { |workflow| workflow.context.success? }
@@ -55,6 +62,7 @@ module Floe
         opt :context, "JSON payload of the Context",              :type => :string
         opt :credentials, "JSON payload with Credentials",        :type => :string
         opt :credentials_file, "Path to a file with Credentials", :type => :string
+        opt :segment_output, "Segment output by each worker",     :default => false
 
         Floe::ContainerRunner.cli_options(self)
 
@@ -82,5 +90,30 @@ module Floe
 
       return workflows_inputs, opts
     end
+
+    def create_credentials(opts)
+      if opts[:credentials_given]
+        opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
+      elsif opts[:credentials_file_given]
+        File.read(opts[:credentials_file])
+      end
+    end
+
+    def create_workflow(workflow, context_payload, input, credentials)
+      context = Floe::Workflow::Context.new(context_payload, :input => input, :credentials => credentials)
+      Floe::Workflow.load(workflow, context)
+    end
+
+    def create_loggers(workflows, segment_output)
+      if workflows.size == 1 || !segment_output
+        # no extra work necessary
+        {}
+      else
+        workflows.each_with_object({}) do |workflow, h|
+          workflow.context.logger = Logger.new(output = StringIO.new)
+          h[workflow] = output
+        end
+      end
+    end
   end
 end

data/lib/floe/container_runner/docker.rb

@@ -30,7 +30,7 @@ module Floe
         end
 
         begin
-          runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"])
+          runner_context["container_ref"] = run_container(image, env, execution_id, runner_context["secrets_ref"], context.logger)
           runner_context
         rescue AwesomeSpawn::CommandResultError => err
           cleanup(runner_context)
@@ -123,7 +123,7 @@ module Floe
 
       attr_reader :network
 
-      def run_container(image, env, execution_id, secrets_file)
+      def run_container(image, env, execution_id, secrets_file, logger)
         params = run_container_params(image, env, execution_id, secrets_file)
 
         logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
@@ -182,8 +182,8 @@ module Floe
 
       def inspect_container(container_id)
         JSON.parse(docker!("inspect", container_id).output).first
-      rescue
-        nil
+      rescue AwesomeSpawn::CommandResultError => err
+        raise Floe::ExecutionError.new("Failed to get status for container #{container_id}: #{err}")
       end
 
       def delete_container(container_id)

data/lib/floe/container_runner/kubernetes.rb

@@ -155,6 +155,8 @@ module Floe
 
       def pod_info(pod_name)
         kubeclient.get_pod(pod_name, namespace)
+      rescue Kubeclient::HttpError => err
+        raise Floe::ExecutionError.new("Failed to get status for pod #{namespace}/#{pod_name}: #{err}")
       end
 
       def pod_running?(context)
@@ -285,7 +287,8 @@ module Floe
         code    = notice.object&.code
         reason  = notice.object&.reason
 
-        logger.warn("Received [#{code} #{reason}], [#{message}]")
+        # This feels like a global concern and not an end user's concern
+        Floe.logger.warn("Received [#{code} #{reason}], [#{message}]")
 
         true
       end

data/lib/floe/logging.rb

@@ -7,7 +7,11 @@ module Floe
     end
 
     def logger
-      Floe.logger
+      @logger || Floe.logger
+    end
+
+    def logger=(logger)
+      @logger = logger
     end
   end
 end

data/lib/floe/runner.rb

@@ -2,8 +2,6 @@
 
 module Floe
   class Runner
-    include Logging
-
     OUTPUT_MARKER = "__FLOE_OUTPUT__\n"
 
     def initialize(_options = {}) # rubocop:disable Style/RedundantInitialize

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.14.0"
+  VERSION = "0.15.1"
 end

data/lib/floe/workflow/branch.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class Branch < Floe::WorkflowBase
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/data.rb

@@ -4,16 +4,20 @@ module Floe
   class Workflow
     class ChoiceRule
       class Data < Floe::Workflow::ChoiceRule
-        COMPARE_KEYS = %w[IsNull IsPresent IsNumeric IsString IsBoolean IsTimestamp String Numeric Boolean Timestamp].freeze
+        TYPES      = ["String", "Numeric", "Boolean", "Timestamp", "Present", "Null"].freeze
+        COMPARES   = ["Equals", "LessThan", "GreaterThan", "LessThanEquals", "GreaterThanEquals", "Matches"].freeze
+        # e.g.: (Is)(String), (Is)(Present)
+        TYPE_CHECK = /^(Is)(#{TYPES.join("|")})$/.freeze
+        # e.g.: (String)(LessThan)(Path), (Numeric)(GreaterThanEquals)()
+        OPERATION  = /^(#{(TYPES - %w[Null Present]).join("|")})(#{COMPARES.join("|")})(Path)?$/.freeze
 
-        attr_reader :variable, :compare_key, :value, :path
+        attr_reader :variable, :compare_key, :operation, :type, :compare_predicate, :path
 
         def initialize(_workflow, _name, payload)
           super
 
-          @variable = parse_path("Variable", payload)
+          @variable = parse_path("Variable")
           parse_compare_key
-          @value = path ? parse_path(compare_key, payload) : payload[compare_key]
         end
 
         def true?(context, input)
@@ -21,39 +25,7 @@ module Floe
 
           lhs = variable_value(context, input)
           rhs = compare_value(context, input)
-
-          case compare_key
-          when "IsNull" then is_null?(lhs, rhs)
-          when "IsNumeric" then is_numeric?(lhs, rhs)
-          when "IsString" then is_string?(lhs, rhs)
-          when "IsBoolean" then is_boolean?(lhs, rhs)
-          when "IsTimestamp" then is_timestamp?(lhs, rhs)
-          when "StringEquals", "StringEqualsPath",
-               "NumericEquals", "NumericEqualsPath",
-               "BooleanEquals", "BooleanEqualsPath",
-               "TimestampEquals", "TimestampEqualsPath"
-            lhs == rhs
-          when "StringLessThan", "StringLessThanPath",
-               "NumericLessThan", "NumericLessThanPath",
-               "TimestampLessThan", "TimestampLessThanPath"
-            lhs < rhs
-          when "StringGreaterThan", "StringGreaterThanPath",
-               "NumericGreaterThan", "NumericGreaterThanPath",
-               "TimestampGreaterThan", "TimestampGreaterThanPath"
-            lhs > rhs
-          when "StringLessThanEquals", "StringLessThanEqualsPath",
-               "NumericLessThanEquals", "NumericLessThanEqualsPath",
-               "TimestampLessThanEquals", "TimestampLessThanEqualsPath"
-            lhs <= rhs
-          when "StringGreaterThanEquals", "StringGreaterThanEqualsPath",
-               "NumericGreaterThanEquals", "NumericGreaterThanEqualsPath",
-               "TimestampGreaterThanEquals", "TimestampGreaterThanEqualsPath"
-            lhs >= rhs
-          when "StringMatches"
-            lhs.match?(Regexp.escape(rhs).gsub('\*', '.*?'))
-          else
-            raise Floe::InvalidWorkflowError, "Invalid choice [#{compare_key}]"
-          end
+          send(operation, lhs, rhs)
         end
 
         private
@@ -108,26 +80,101 @@ module Floe
         # rubocop:enable Naming/PredicateName
         # rubocop:enable Style/OptionalBooleanParameter
 
+        def equals?(lhs, rhs)
+          lhs == rhs
+        end
+
+        def lessthan?(lhs, rhs)
+          lhs < rhs
+        end
+
+        def greaterthan?(lhs, rhs)
+          lhs > rhs
+        end
+
+        def lessthanequals?(lhs, rhs)
+          lhs <= rhs
+        end
+
+        def greaterthanequals?(lhs, rhs)
+          lhs >= rhs
+        end
+
+        def matches?(lhs, rhs)
+          lhs.match?(Regexp.escape(rhs).gsub('\*', '.*?'))
+        end
+
+        # parse the compare key at initialization time
         def parse_compare_key
-          @compare_key = payload.keys.detect { |key| key.match?(/^(#{COMPARE_KEYS.join("|")})/) }
-          parser_error!("requires a compare key") unless compare_key
+          payload.each_key do |key|
+            # e.g. (String)(GreaterThan)(Path)
+            if (match_values = OPERATION.match(key))
+              @compare_key = key
+              @type, operator, @path = match_values.captures
+              @operation = "#{operator.downcase}?".to_sym
+              @compare_predicate = parse_predicate(type)
+              break
+            end
+            # e.g. (Is)(String)
+            if (match_value = TYPE_CHECK.match(key))
+              @compare_key = key
+              _operator, type = match_value.captures
+              # type: nil means no runtime type checking.
+              @type = @path = nil
+              @operation = "is_#{type.downcase}?".to_sym
+              @compare_predicate = parse_predicate("Boolean")
+              break
+            end
+          end
+          parser_error!("requires a compare key") if compare_key.nil? || operation.nil?
+        end
 
-          @path = compare_key.end_with?("Path")
+        # parse predicate at initilization time
+        # @return the right predicate attached to the compare key
+        def parse_predicate(data_type)
+          path ? parse_path(compare_key) : parse_field(compare_key, data_type)
         end
 
+        # @return right hand predicate - input path or static payload value)
         def compare_value(context, input)
-          path ? value.value(context, input) : value
+          path ? fetch_path(compare_key, compare_predicate, context, input) : compare_predicate
         end
 
+        # feth the variable value at runtime
+        # @return variable value (left hand side )
         def variable_value(context, input)
-          variable.value(context, input)
+          fetch_path("Variable", variable, context, input)
         end
 
-        def parse_path(field_name, payload)
+        # parse path at initilization time
+        # helper method to parse a path from the payload
+        def parse_path(field_name)
           value = payload[field_name]
           missing_field_error!(field_name) unless value
           wrap_parser_error(field_name, value) { Path.new(value) }
         end
+
+        # parse predicate field at initialization time
+        def parse_field(field_name, data_type)
+          value = payload[field_name]
+          return value if correct_type?(value, data_type)
+
+          invalid_field_error!(field_name, value, "required to be a #{data_type}")
+        end
+
+        # fetch a path at runtime
+        def fetch_path(field_name, field_path, context, input)
+          value = field_path.value(context, input)
+          return value if type.nil? || correct_type?(value, type)
+
+          runtime_field_error!(field_name, field_path.to_s, "required to point to a #{type}")
+        end
+
+        # if we have runtime checking, check against that type
+        #   otherwise assume checking a TYPE_CHECK predicate and check against Boolean
+        def correct_type?(value, data_type)
+          send("is_#{data_type.downcase}?".to_sym, value)
+        end
       end
     end
   end

data/lib/floe/workflow/context.rb

@@ -3,11 +3,13 @@
 module Floe
   class Workflow
     class Context
+      include Logging
+
       attr_accessor :credentials
 
       # @param context [Json|Hash] (default, create another with input and execution params)
       # @param input [Hash] (default: {})
-      def initialize(context = nil, input: nil, credentials: {})
+      def initialize(context = nil, input: nil, credentials: {}, logger: nil)
         context = JSON.parse(context) if context.kind_of?(String)
         input   = JSON.parse(input || "{}")
 
@@ -20,6 +22,8 @@ module Floe
         self["Task"]               ||= {}
 
         @credentials = credentials || {}
+
+        self.logger = logger if logger
       rescue JSON::ParserError => err
         raise Floe::InvalidExecutionInput, "Invalid State Machine Execution Input: #{err}: was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
       end

data/lib/floe/workflow/item_processor.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ItemProcessor < Floe::WorkflowBase
+      attr_reader :processor_config
+
+      def initialize(payload, name = nil)
+        super
+        @processor_config = payload.fetch("ProcessorConfig", "INLINE")
+      end
+    end
+  end
+end

data/lib/floe/workflow/state.rb

@@ -3,7 +3,6 @@
 module Floe
   class Workflow
     class State
-      include Logging
       include ValidationMixin
 
       class << self
@@ -56,14 +55,22 @@ module Floe
         mark_started(context)
       end
 
+      def started?(context)
+        context.state_started?
+      end
+
       def finish(context)
         mark_finished(context)
       end
 
+      def finished?(context)
+        context.state_finished?
+      end
+
       def mark_started(context)
         context.state["EnteredTime"] = Time.now.utc.iso8601
 
-        logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...")
+        context.logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...")
       end
 
       def mark_finished(context)
@@ -74,7 +81,7 @@ module Floe
         context.state["Duration"]       = finished_time - entered_time
 
         level = context.failed? ? :error : :info
-        logger.public_send(level, "Running state: [#{long_name}] with input [#{context.json_input}]...Complete #{context.next_state ? "- next state [#{context.next_state}]" : "workflow -"} output: [#{context.json_output}]")
+        context.logger.public_send(level, "Running state: [#{long_name}] with input [#{context.json_input}]...Complete #{context.next_state ? "- next state [#{context.next_state}]" : "workflow -"} output: [#{context.json_output}]")
 
         0
       end
@@ -88,7 +95,7 @@ module Floe
       end
 
       def ready?(context)
-        !context.state_started? || !running?(context)
+        !started?(context) || !running?(context)
       end
 
       def running?(context)