floe 0.15.1 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f21cf2fcc6a7fa9f7536e0cca328d101a867c389f53f768e4746b6e69f643a5
-  data.tar.gz: f3d82401842f66504afdeaa4b051db01d5fcb9e79b704af368712d68b4950251
+  metadata.gz: 2b11d410b8a7518f7030bac6a5a29142167f54239e7cf1f966c6a82c2a191207
+  data.tar.gz: 793dd3f55b6d89b22d335831ca0a8663015ad520e8cf92603884f9168a64c8f6
 SHA512:
-  metadata.gz: 3e73dbdd868ec57b8138a9e4b0b529c0f2c44cdb21d43c0ae29ca9094512f4044cc621a38e4ac217bf6b17e13f1ef3a361cf5917023eab87d30c7b4cc90eccfc
-  data.tar.gz: e174fabde62d6348f8517f7c2d019600b34612f5b8d822eb83c2c5579f5e224596b6edaec383bfe9cce75c3260e50a4a0b744764119cd96e2a973f92b49b7d5a
+  metadata.gz: c495de319815ea25d470262472e31675d016d5aa8d6ca761e64eb94556617809e7311b4dcce2242f1ccf0910c893679e44729357dca99b6b9b5299dff2fee0db
+  data.tar.gz: bcfdf7696caeaf80d6464dcc375747a40127c8327d1a75e486ad56e5d9618ef4f6ea73a3a5c35bb11f0f0a8c1e844b208b4a676f4555142cd7047c021d23dccf

data/CHANGELOG.md

@@ -4,6 +4,28 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.17.0] - 2025-08-19
+### Fixed
+- Fix credentials passed via CLI ([#307](https://github.com/ManageIQ/floe/pull/307))
+- Fix nested PayloadTemplate interpolation ([#311](https://github.com/ManageIQ/floe/pull/311))
+
+  NOTE: Using `".$"` for hash keys is no longer required and will result in an error
+
+### Changed
+- Allow credentials to be referenced from parameters ([#308](https://github.com/ManageIQ/floe/pull/308))
+
+  NOTE: Setting credentials via ResultPath now uses `$$.Credentials`
+
+### Added
+- Add floe:// builtin runner and floe://http method ([#306](https://github.com/ManageIQ/floe/pull/306))
+
+## [0.16.0] - 2025-04-08
+### Added
+- Add Map state ItemBatcher/ItemSelector support ([#294](https://github.com/ManageIQ/floe/pull/294))
+
+### Fixed
+- Fix JSON.parse exception expectation ([#300](https://github.com/ManageIQ/floe/pull/300))
+
 ## [0.15.1] - 2024-11-21
 ### Fixed
 - Fix Map/Parallel States checking container_runner#status! of finished states ([#296](https://github.com/ManageIQ/floe/pull/296))
@@ -272,7 +294,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.15.1...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.17.0...HEAD
+[0.16.0]: https://github.com/ManageIQ/floe/compare/v0.16.0...v0.17.0
+[0.16.0]: https://github.com/ManageIQ/floe/compare/v0.15.1...v0.16.0
 [0.15.1]: https://github.com/ManageIQ/floe/compare/v0.15.0...v0.15.1
 [0.15.0]: https://github.com/ManageIQ/floe/compare/v0.14.0...v0.15.0
 [0.14.0]: https://github.com/ManageIQ/floe/compare/v0.13.1...v0.14.0

data/README.md

@@ -61,7 +61,7 @@ Or you can pass a file path with the `--credentials-file` parameter:
 bundle exec ruby exe/floe --workflow my-workflow.asl --credentials-file /tmp/20231218-80537-kj494t
 ```
 
-If you need to set a credential at runtime you can do that by using the `"ResultPath": "$.Credentials"` directive, for example to user a username/password to login and get a Bearer token:
+If you need to set a credential at runtime you can do that by using the `"ResultPath": "$$.Credentials"` directive, for example to user a username/password to login and get a Bearer token:
 
 ```
 bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"username": "user", "password": "pass"}'
@@ -78,14 +78,14 @@ bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"username":
         "username.$": "$.username",
         "password.$": "$.password"
       },
-      "ResultPath": "$.Credentials",
+      "ResultPath": "$$.Credentials",
       "Next": "DoSomething"
     },
     "DoSomething": {
       "Type": "Task",
       "Resource": "docker://do-something:latest",
       "Credentials": {
-        "token.$": "$.bearer_token"
+        "token.$": "$$.Credentials.bearer_token"
       },
       "End": true
     }
@@ -155,6 +155,78 @@ until running_workflows.empty?
 end
 ```
 
+### Task Runners
+
+Floe provides a number of options for the Task `"Resource"` parameter.  The `"Resource"` runner is declared by the author based on the `URI`.
+
+Task Runner Types:
+* `floe://`
+* `docker://`
+
+#### Floe resource
+
+This is the "builtin" runner and exposes methods that are executed internally without having to call out to a docker image.
+
+##### HTTP builtin method
+
+`floe://http` allows you to execute HTTP method calls.
+
+Example:
+```json
+{
+  "Comment": "Execute a HTTP call",
+  "StartAt": "HTTP",
+  "States": {
+    "HTTP": {
+      "Type": "Task",
+      "Resource": "floe://http",
+      "Parameters": {
+        "Method": "POST",
+        "Url": "http://localhost:3000/api/login",
+        "Headers": {"ContentType": "application/json"},
+        "Body.$": {"username.$": "$$.Credentials.username", "password.$": "$$.Credentials.password"},
+        "Options": {"Encoding": "JSON"}
+      },
+      "ResultSelector": {"auth_token.$": "$.Body.auth_token"},
+      "ResultPath": "$$.Credentials",
+      "End": true
+    }
+```
+
+HTTP Parameters:
+* `Method` (required) - HTTP method name. Permitted values: `GET`, `POST`, `PUT`, `DELETE`, `HEAD`, `PATCH`, `OPTIONS`, or `TRACE`
+* `Url` (required) - URL to execute the HTTP call to
+* `Headers` - Hash of unencoded HTTP request header key/value pairs.
+* `QueryParameters` - URI query unencoded key/value pairs.
+* `Body` - HTTP request body.  Depending on Encoding this can be a String or a Hash of key/value pairs.
+* `Ssl` - SSL options
+  * `Verify` - Boolean - Verify SSL certificate.
+  * `VerifyHostname` - Boolean - Verify SSL certificate hostname.
+  * `Hostname` - String - Server hostname for SNI.
+  * `CaFile` - String - Path to a CA file in PEM format.
+  * `CaPath` - String - Path to a CA directory.
+  * `VerifyMode` - Integer - OpenSSL constant.
+  * `VerifyDepth` - Integer - Maximum depth for the certificate chain validation.
+  * `Version` - Integer - SSL Version.
+  * `MinVersion` - Integer - Minimum SSL Version.
+  * `MaxVersion` - Integer - Maximum SSL Version.
+  * `Ciphers` - String - Ciphers supported.
+* `Options`
+  * `Timeout`
+  * `ReadTimeout`
+  * `OpenTimeout`
+  * `WriteTimeout`
+  * `Encoding` - String
+    * `JSON` - JSON encodes the request and decodes the response
+* `Proxy`
+  * `Uri` - String - URI of the proxy.
+  * `User` - String - User for the proxy.
+  * `Password` - String - Pasword for the proxy
+
+#### Docker resource
+
+The docker resource runner takes a docker image URI, including the registry, name, and tag.
+
 ### Docker Runner Options
 
 #### Docker

data/examples/everything.asl

@@ -0,0 +1,146 @@
+{
+  "Comment": "An example of the Amazon States Language with all states.",
+  "StartAt": "FirstState",
+  "States": {
+    "FirstState": {
+      "Type": "Task",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
+      "Credentials": {
+        "mysecret": "dont tell anyone"
+      },
+      "Retry": [
+        {
+          "ErrorEquals": [ "States.Timeout" ],
+          "IntervalSeconds": 3,
+          "MaxAttempts": 2,
+          "BackoffRate": 1.5
+        }
+      ],
+      "Catch": [
+        {
+          "ErrorEquals": [ "States.ALL" ],
+          "Next": "FailState"
+        }
+      ],
+      "Next": "ChoiceState"
+    },
+
+    "ChoiceState": {
+      "Type" : "Choice",
+      "Choices": [
+        {
+          "Variable": "$.foo",
+          "NumericEquals": 1,
+          "Next": "FirstMatchState"
+        },
+        {
+          "Variable": "$.foo",
+          "NumericEquals": 2,
+          "Next": "SecondMatchState"
+        },
+        {
+          "Variable": "$.foo",
+          "NumericEquals": 3,
+          "Next": "SuccessState"
+        }
+      ],
+      "Default": "FailState"
+    },
+
+    "FirstMatchState": {
+      "Type" : "Task",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
+      "Next": "PassState"
+    },
+
+    "SecondMatchState": {
+      "Type" : "Task",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
+      "Next": "WaitState"
+    },
+
+    "WaitState": {
+      "Type": "Wait",
+      "Seconds": 1,
+      "Next": "PassState"
+    },
+
+    "PassState": {
+      "Type": "Pass",
+      "Next": "MapState",
+      "Result": {
+        "foo": "bar",
+        "colors": [
+          "red",
+          "green",
+          "blue",
+          "yellow",
+          "white"
+        ]
+      }
+    },
+
+    "FailState": {
+      "Type": "Fail",
+      "Error": "FailStateError",
+      "Cause": "No Matches!"
+    },
+
+    "MapState": {
+      "Type": "Map",
+      "ItemsPath": "$.colors",
+      "MaxConcurrency": 2,
+      "ItemProcessor": {
+        "ProcessorConfig": {
+          "Mode": "INLINE"
+        },
+        "StartAt": "Generate UUID",
+        "States": {
+          "Generate UUID": {
+            "Type": "Pass",
+            "Next": "PassState",
+            "Parameters": {
+              "uuid.$": "States.UUID()"
+            }
+          },
+          "PassState": {
+            "Type": "Pass",
+            "End": true
+          }
+        }
+      },
+      "Next": "ParallelState"
+    },
+
+    "ParallelState": {
+      "Type": "Parallel",
+      "Next": "SuccessState",
+      "Branches": [
+        {
+          "StartAt": "Add",
+          "States": {
+            "Add": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        },
+        {
+          "StartAt": "Subtract",
+          "States": {
+            "Subtract": {
+              "Type": "Task",
+              "Resource": "docker://docker.io/agrare/sleep:latest",
+              "End": true
+            }
+          }
+        }
+      ]
+    },
+
+    "SuccessState": {
+      "Type": "Succeed"
+    }
+  }
+}

data/examples/http.asl

@@ -0,0 +1,40 @@
+{
+  "Comment": "Execute a REST API call",
+  "StartAt": "Login",
+  "States": {
+    "Login": {
+      "Type": "Task",
+      "Resource": "floe://http",
+      "Parameters": {
+        "Method": "GET",
+        "Url": "http://localhost:3000/api/auth",
+        "Headers": {
+          "ContentType": "application/json",
+          "Authorization.$": "States.Format('Basic {}', States.Base64Encode(States.Format('{}:{}', $$.Credentials.username, $$.Credentials.password)))"
+        },
+        "Options": {"Encoding": "JSON"}
+      },
+      "ResultSelector": {"auth_token.$": "$.Body.auth_token"},
+      "ResultPath": "$$.Credentials",
+      "Next": "List VMs"
+    },
+    "List VMs": {
+      "Type": "Task",
+      "Resource": "floe://http",
+      "Parameters": {
+        "Method": "GET",
+        "Url": "http://localhost:3000/api/vms",
+        "Headers": {
+          "ContentType": "application/json",
+          "X-Auth-Token.$": "$$.Credentials.auth_token"
+        },
+        "Options": {"Encoding": "JSON"}
+      },
+      "ResultSelector": {
+        "Status.$": "$.Status",
+        "Resources.$": "$.Body.resources"
+      },
+      "End": true
+    }
+  }
+}

data/examples/set-credential.asl

@@ -8,7 +8,7 @@
       "Parameters": {
         "ECHO": "TOKEN"
       },
-      "ResultPath": "$.Credentials",
+      "ResultPath": "$$.Credentials",
       "ResultSelector": {
         "bearer_token.$": "$.echo"
       },

data/floe.gemspec

@@ -11,7 +11,7 @@ Gem::Specification.new do |spec|
   spec.description = spec.summary
   spec.homepage = "https://github.com/ManageIQ/floe"
   spec.licenses = ["Apache-2.0"]
-  spec.required_ruby_version = ">= 2.7.0"
+  spec.required_ruby_version = ">= 3.0.0"
 
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
   spec.metadata['rubygems_mfa_required'] = "true"
@@ -37,6 +37,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "optimist", "~>3.0"
   spec.add_dependency "parslet", "~>2.0"
+  spec.add_dependency "json", "~>2.10"
+  spec.add_dependency "faraday"
+  spec.add_dependency "faraday-follow_redirects"
 
   spec.add_development_dependency "manageiq-style", ">= 1.5.2"
   spec.add_development_dependency "rake", "~> 13.0"

data/lib/floe/builtin_runner/methods.rb

@@ -0,0 +1,82 @@
+module Floe
+  module BuiltinRunner
+    class Methods < BasicObject
+      def self.http(params, _secrets, _context)
+        error = http_verify_params(params)
+        return BuiltinRunner.error!({}, :cause => error) if error
+
+        method, url, headers, query, body =
+          params.values_at("Method", "Url", "Headers", "QueryParameters", "Body")
+
+        ssl = {
+          "verify"          => params.dig("Ssl", "Verify"),
+          "verify_hostname" => params.dig("Ssl", "VerifyHostname"),
+          "hostname"        => params.dig("Ssl", "Hostname"),
+          "ca_file"         => params.dig("Ssl", "CaFile"),
+          "ca_path"         => params.dig("Ssl", "CaPath"),
+          "verify_mode"     => params.dig("Ssl", "VerifyMode"),
+          "verify_depth"    => params.dig("Ssl", "VerifyDepth"),
+          "version"         => params.dig("Ssl", "Version"),
+          "min_version"     => params.dig("Ssl", "MinVersion"),
+          "max_version"     => params.dig("Ssl", "MaxVersion"),
+          "ciphers"         => params.dig("Ssl", "Ciphers")
+        }.compact
+
+        request = {
+          "timeout"       => params.dig("Options", "Timeout"),
+          "read_timeout"  => params.dig("Options", "ReadTimeout"),
+          "open_timeout"  => params.dig("Options", "OpenTimeout"),
+          "write_timeout" => params.dig("Options", "WriteTimeout")
+        }.compact
+
+        proxy = {
+          "uri"      => params.dig("Proxy", "Uri"),
+          "user"     => params.dig("Proxy", "User"),
+          "password" => params.dig("Proxy", "Password")
+        }.compact
+
+        connection_options = {
+          :url     => url,
+          :params  => query,
+          :headers => headers,
+          :request => (request unless request.empty?),
+          :proxy   => (proxy   unless proxy.empty?),
+          :ssl     => (ssl     unless ssl.empty?)
+        }
+
+        require "faraday"
+        connection = ::Faraday.new(connection_options)
+
+        if params.dig("Options", "Encoding") == "JSON"
+          connection.request(:json)
+          connection.response(:json)
+        end
+
+        if params.dig("Options", "FollowRedirects") != false
+          require "faraday/follow_redirects"
+          connection.response(:follow_redirects)
+        end
+
+        response = connection.send(method.downcase) do |request|
+          request.body = body if body
+        end
+
+        output = {"Status" => response.status, "Body" => response.body, "Headers" => response.headers}
+
+        BuiltinRunner.success!({}, :output => output)
+      end
+
+      private_class_method def self.http_verify_params(params)
+        return "Missing Parameter: Url"    if params["Url"].nil?
+        return "Missing Parameter: Method" if params["Method"].nil?
+        return "Invalid Parameter: Method: [#{params["Method"]}], must be GET, POST, PUT, DELETE, HEAD, PATCH, OPTIONS, or TRACE" unless %w[GET POST PUT DELETE HEAD PATCH OPTIONS TRACE].include?(params["Method"])
+
+        nil
+      end
+
+      private_class_method def self.http_status!(runner_context)
+        runner_context
+      end
+    end
+  end
+end

data/lib/floe/builtin_runner/runner.rb

@@ -0,0 +1,53 @@
+module Floe
+  module BuiltinRunner
+    class Runner < Floe::Runner
+      def run_async!(resource, params, secrets, context)
+        raise ArgumentError, "Invalid resource" unless resource&.start_with?(SCHEME_PREFIX)
+
+        method_name = resource.sub(SCHEME_PREFIX, "")
+
+        begin
+          runner_context = {"method" => method_name}
+          method_result = Methods.public_send(method_name, params, secrets, context)
+          method_result.merge(runner_context)
+        rescue NoMethodError
+          Floe::BuiltinRunner.error!(runner_context, :cause => "undefined method [#{method_name}]")
+        rescue => err
+          Floe::BuiltinRunner.error!(runner_context, :cause => err.to_s)
+        ensure
+          cleanup(runner_context)
+        end
+      end
+
+      def cleanup(runner_context)
+        method_name = runner_context["method"]
+        raise ArgumentError if method_name.nil?
+
+        cleanup_method = "#{method_name}_cleanup"
+        return unless Methods.respond_to?(cleanup_method, true)
+
+        Methods.send(cleanup_method, runner_context)
+      end
+
+      def status!(runner_context)
+        method_name = runner_context["method"]
+        raise ArgumentError if method_name.nil?
+        return if runner_context["running"] == false
+
+        Methods.send("#{method_name}_status!", runner_context)
+      end
+
+      def running?(runner_context)
+        runner_context["running"]
+      end
+
+      def success?(runner_context)
+        runner_context["success"]
+      end
+
+      def output(runner_context)
+        runner_context["output"]
+      end
+    end
+  end
+end

data/lib/floe/builtin_runner.rb

@@ -0,0 +1,25 @@
+require "floe/builtin_runner/runner"
+require "floe/builtin_runner/methods"
+
+module Floe
+  module BuiltinRunner
+    SCHEME        = "floe".freeze
+    SCHEME_PREFIX = "#{SCHEME}://".freeze
+
+    class << self
+      def error!(runner_context = {}, cause:, error: "States.TaskFailed")
+        runner_context.merge!(
+          "running" => false, "success" => false, "output" => {"Error" => error, "Cause" => cause}
+        )
+      end
+
+      def success!(runner_context = {}, output:)
+        runner_context.merge!(
+          "running" => false, "success" => true, "output" => output
+        )
+      end
+    end
+  end
+end
+
+Floe::Runner.register_scheme(Floe::BuiltinRunner::SCHEME, -> { Floe::BuiltinRunner::Runner.new })

data/lib/floe/cli.rb

@@ -92,11 +92,14 @@ module Floe
     end
 
     def create_credentials(opts)
-      if opts[:credentials_given]
-        opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
-      elsif opts[:credentials_file_given]
-        File.read(opts[:credentials_file])
-      end
+      credentials_str = if opts[:credentials_given]
+                          opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
+                        elsif opts[:credentials_file_given]
+                          File.read(opts[:credentials_file])
+                        else
+                          return
+                        end
+      JSON.parse(credentials_str)
     end
 
     def create_workflow(workflow, context_payload, input, credentials)

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.15.1"
+  VERSION = "0.17.0"
 end

data/lib/floe/workflow/choice_rule/data.rb

@@ -80,27 +80,27 @@ module Floe
         # rubocop:enable Naming/PredicateName
         # rubocop:enable Style/OptionalBooleanParameter
 
-        def equals?(lhs, rhs)
+        def op_equals?(lhs, rhs)
           lhs == rhs
         end
 
-        def lessthan?(lhs, rhs)
+        def op_lessthan?(lhs, rhs)
           lhs < rhs
         end
 
-        def greaterthan?(lhs, rhs)
+        def op_greaterthan?(lhs, rhs)
           lhs > rhs
         end
 
-        def lessthanequals?(lhs, rhs)
+        def op_lessthanequals?(lhs, rhs)
           lhs <= rhs
         end
 
-        def greaterthanequals?(lhs, rhs)
+        def op_greaterthanequals?(lhs, rhs)
           lhs >= rhs
         end
 
-        def matches?(lhs, rhs)
+        def op_matches?(lhs, rhs)
           lhs.match?(Regexp.escape(rhs).gsub('\*', '.*?'))
         end
 
@@ -111,7 +111,7 @@ module Floe
             if (match_values = OPERATION.match(key))
               @compare_key = key
               @type, operator, @path = match_values.captures
-              @operation = "#{operator.downcase}?".to_sym
+              @operation = "op_#{operator.downcase}?".to_sym
               @compare_predicate = parse_predicate(type)
               break
             end

data/lib/floe/workflow/context.rb

@@ -5,15 +5,14 @@ module Floe
     class Context
       include Logging
 
-      attr_accessor :credentials
-
       # @param context [Json|Hash] (default, create another with input and execution params)
       # @param input [Hash] (default: {})
-      def initialize(context = nil, input: nil, credentials: {}, logger: nil)
+      def initialize(context = nil, input: nil, credentials: nil, logger: nil)
         context = JSON.parse(context) if context.kind_of?(String)
         input   = JSON.parse(input || "{}")
 
         @context = context || {}
+        self["Credentials"]        ||= credentials || {}
         self["Execution"]          ||= {}
         self["Execution"]["Input"] ||= input
         self["State"]              ||= {}
@@ -21,8 +20,6 @@ module Floe
         self["StateMachine"]       ||= {}
         self["Task"]               ||= {}
 
-        @credentials = credentials || {}
-
         self.logger = logger if logger
       rescue JSON::ParserError => err
         raise Floe::InvalidExecutionInput, "Invalid State Machine Execution Input: #{err}: was expecting (JSON String, Number, Array, Object or token 'null', 'true' or 'false')"
@@ -32,6 +29,10 @@ module Floe
         @context["Execution"]
       end
 
+      def credentials
+        @context["Credentials"]
+      end
+
       def started?
         execution.key?("StartTime")
       end
@@ -138,8 +139,18 @@ module Floe
         @context.dig(*args)
       end
 
+      def inspect
+        format("#<%s: %s>", self.class.name, safe_context.inspect)
+      end
+
       def to_h
-        @context
+        safe_context
+      end
+
+      private
+
+      def safe_context
+        @context.except("Credentials")
       end
     end
   end

data/lib/floe/workflow/item_batcher.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ItemBatcher
+      include ValidationMixin
+
+      attr_reader :name, :batch_input, :max_items_per_batch, :max_items_per_batch_path, :max_input_bytes_per_batch, :max_input_bytes_per_batch_path
+
+      def initialize(payload, name)
+        @name = name
+
+        @batch_input               = PayloadTemplate.new(payload["BatchInput"]) if payload["BatchInput"]
+        @max_items_per_batch       = payload["MaxItemsPerBatch"]
+        @max_input_bytes_per_batch = payload["MaxInputBytesPerBatch"]
+
+        @max_items_per_batch_path       = ReferencePath.new(payload["MaxItemsPerBatchPath"])      if payload["MaxItemsPerBatchPath"]
+        @max_input_bytes_per_batch_path = ReferencePath.new(payload["MaxInputBytesPerBatchPath"]) if payload["MaxInputBytesPerBatchPath"]
+
+        validate!
+      end
+
+      def value(context, input, state_input = nil)
+        state_input ||= input
+
+        output = batch_input ? batch_input.value(context, state_input) : {}
+
+        input.each_slice(max_items(context, state_input)).map do |batch|
+          output.merge("Items" => batch)
+        end
+      end
+
+      private
+
+      def max_items(context, state_input)
+        return    max_items_per_batch if max_items_per_batch
+        return if max_items_per_batch_path.nil?
+
+        result = max_items_per_batch_path.value(context, state_input)
+        raise runtime_field_error!("MaxItemsPerBatchPath", result, "must be a positive integer") if result.nil? || !result.kind_of?(Integer) || result <= 0
+
+        result
+      end
+
+      def validate!
+        if [max_items_per_batch, max_items_per_batch_path].all?(&:nil?)
+          parser_error!("must have one of \"MaxItemsPerBatch\", \"MaxItemsPerBatchPath\"")
+        end
+
+        parser_error!("must not specify both \"MaxItemsPerBatch\" and \"MaxItemsPerBatchPath\"")           if max_items_per_batch && max_items_per_batch_path
+        parser_error!("must not specify both \"MaxInputBytesPerBatch\" and \"MaxInputBytesPerBatchPath\"") if max_input_bytes_per_batch && max_input_bytes_per_batch_path
+
+        if max_items_per_batch && (!max_items_per_batch.kind_of?(Integer) || max_items_per_batch <= 0)
+          invalid_field_error!("MaxItemsPerBatch", max_items_per_batch, "must be a positive integer")
+        end
+        if max_input_bytes_per_batch && (!max_input_bytes_per_batch.kind_of?(Integer) || max_input_bytes_per_batch <= 0)
+          invalid_field_error!("MaxInputBytesPerBatch", max_input_bytes_per_batch, "must be a positive integer")
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/payload_template.rb

@@ -17,11 +17,9 @@ module Floe
 
       def parse_payload(value)
         case value
-        when Array  then parse_payload_array(value)
-        when Hash   then parse_payload_hash(value)
-        when String then parse_payload_string(value)
-        else
-          value
+        when Array then parse_payload_array(value)
+        when Hash  then parse_payload_hash(value)
+        else value
         end
       end
 
@@ -32,29 +30,40 @@ module Floe
       def parse_payload_hash(value)
         value.to_h do |key, val|
           if key.end_with?(".$")
-            check_key_conflicts(key, value)
+            check_dynamic_datatype(key, val)
+            check_dynamic_key_conflicts(key, value)
 
-            [key, parse_payload(val)]
+            [key, parse_dynamic_payload_string(key, val)]
           else
-            [key, val]
+            [key, parse_payload(val)]
           end
         end
       end
 
-      def parse_payload_string(value)
+      def parse_dynamic_payload_string(key, value)
         return Path.new(value)              if Path.path?(value)
         return IntrinsicFunction.new(value) if IntrinsicFunction.intrinsic_function?(value)
 
-        value
+        raise Floe::InvalidWorkflowError, "The value for the field \"#{key}\" must be a String that contains a valid Reference Path or Intrinsic Function expression"
+      end
+
+      def check_dynamic_datatype(key, value)
+        unless value.is_a?(String)
+          raise Floe::InvalidWorkflowError, "The value for the field \"#{key}\" must be a String that contains a valid Reference Path or Intrinsic Function expression"
+        end
+      end
+
+      def check_dynamic_key_conflicts(key, value)
+        if value.key?(key.chomp(".$"))
+          raise Floe::InvalidWorkflowError, "both #{key} and #{key.chomp(".$")} present"
+        end
       end
 
       def interpolate_value(value, context, inputs)
         case value
-        when Array                   then interpolate_value_array(value, context, inputs)
-        when Hash                    then interpolate_value_hash(value, context, inputs)
-        when Path, IntrinsicFunction then value.value(context, inputs)
-        else
-          value
+        when Array then interpolate_value_array(value, context, inputs)
+        when Hash  then interpolate_value_hash(value, context, inputs)
+        else value
         end
       end
 
@@ -65,17 +74,16 @@ module Floe
       def interpolate_value_hash(value, context, inputs)
         value.to_h do |key, val|
           if key.end_with?(".$")
-            [key.chomp(".$"), interpolate_value(val, context, inputs)]
+            [key.chomp(".$"), interpolate_dynamic_value(val, context, inputs)]
           else
-            [key, val]
+            [key, interpolate_value(val, context, inputs)]
           end
         end
       end
 
-      def check_key_conflicts(key, value)
-        if value.key?(key.chomp(".$"))
-          raise Floe::InvalidWorkflowError, "both #{key} and #{key.chomp(".$")} present"
-        end
+      def interpolate_dynamic_value(value, context, inputs)
+        # value will be a Path or IntrinsicFunction
+        value.value(context, inputs)
       end
     end
   end

data/lib/floe/workflow/reference_path.rb

@@ -8,10 +8,13 @@ module Floe
       def initialize(*)
         super
 
-        raise Floe::InvalidWorkflowError, "Invalid Reference Path" if payload.match?(/@|,|:|\?/)
+        raise Floe::InvalidWorkflowError, "Invalid Reference Path"              if payload.match?(/@|,|:|\?/)
+        raise Floe::InvalidWorkflowError, "Reference Path cannot start with $$" if payload.start_with?("$$") && !payload.start_with?("$$.Credentials")
+
+        path_shift = payload.start_with?("$$.Credentials") ? 3 : 1
 
         @path = JsonPath.new(payload)
-                        .path[1..]
+                        .path[path_shift..]
                         .map { |v| v.match(/\[(?<name>.+)\]/)["name"] }
                         .filter_map { |v| v[0] == "'" ? v.delete("'") : v.to_i }
       end

data/lib/floe/workflow/states/child_workflow_mixin.rb

@@ -49,7 +49,7 @@ module Floe
         end
 
         def each_child_context(context)
-          context.state[child_context_key].map { |ctx| Context.new(ctx) }
+          context.state[child_context_key].map { |ctx| Context.new(ctx, :credentials => context.credentials) }
         end
       end
     end

data/lib/floe/workflow/states/input_output_mixin.rb

@@ -15,9 +15,8 @@ module Floe
           return if output_path.nil?
 
           results = result_selector.value(context, results) if @result_selector
-          if result_path.payload.start_with?("$.Credentials")
-            credentials = result_path.set(context.credentials, results)["Credentials"]
-            context.credentials.merge!(credentials)
+          if result_path.payload.match?(/^\$\$\.Credentials\b/)
+            context.credentials.merge!(result_path.set(context.credentials, results))
             output = context.input.dup
           else
             output = result_path.set(context.input.dup, results)

data/lib/floe/workflow/states/map.rb

@@ -31,8 +31,8 @@ module Floe
           @item_processor  = ItemProcessor.new(payload["ItemProcessor"], name)
           @items_path      = ReferencePath.new(payload.fetch("ItemsPath", "$"))
           @item_reader     = payload["ItemReader"]
-          @item_selector   = payload["ItemSelector"]
-          @item_batcher    = payload["ItemBatcher"]
+          @item_selector   = PayloadTemplate.new(payload["ItemSelector"]) if payload["ItemSelector"]
+          @item_batcher    = ItemBatcher.new(payload["ItemBatcher"], name + ["ItemBatcher"]) if payload["ItemBatcher"]
           @result_writer   = payload["ResultWriter"]
           @max_concurrency = payload["MaxConcurrency"]&.to_i
           @tolerated_failure_percentage = payload["ToleratedFailurePercentage"]&.to_i
@@ -43,7 +43,9 @@ module Floe
 
         def process_input(context)
           input = super
-          items_path.value(context, input)
+          input = items_path.value(context, input)
+          input = item_batcher.value(context, input, context.state["Input"]) if item_batcher
+          input
         end
 
         def start(context)
@@ -51,7 +53,20 @@ module Floe
 
           input = process_input(context)
 
-          context.state["ItemProcessorContext"] = input.map { |item| Context.new({"Execution" => {"Id" => context.execution["Id"]}}, :input => item.to_json).to_h }
+          context.state["ItemProcessorContext"] = input.map.with_index do |item, index|
+            item_processor_context = {
+              "Execution" => {
+                "Id" => context.execution["Id"]
+              },
+              "Map" => {
+                "Item" => {"Index" => index, "Value" => item}
+              }
+            }
+
+            item_processor_input = item_selector ? item_selector.value(item_processor_context, context.state["Input"]) : item
+
+            Context.new(item_processor_context, :input => item_processor_input.to_json).to_h
+          end
         end
 
         def end?

data/lib/floe/workflow/states/task.rb

@@ -40,7 +40,8 @@ module Floe
           super
 
           input          = process_input(context)
-          runner_context = runner.run_async!(resource, input, credentials&.value({}, context.credentials), context)
+          secrets        = credentials&.value(context, context.input)
+          runner_context = runner.run_async!(resource, input, secrets, context)
 
           context.state["RunnerContext"] = runner_context
         end

data/lib/floe/workflow.rb

@@ -8,7 +8,7 @@ module Floe
     include Logging
 
     class << self
-      def load(path_or_io, context = nil, credentials = {}, name = nil)
+      def load(path_or_io, context = nil, credentials = nil, name = nil)
         payload = path_or_io.respond_to?(:read) ? path_or_io.read : File.read(path_or_io)
         # default the name if it is a filename and none was passed in
         name ||= path_or_io.respond_to?(:read) ? "stream" : path_or_io.split("/").last.split(".").first

data/lib/floe.rb

@@ -6,6 +6,7 @@ require_relative "floe/null_logger"
 require_relative "floe/logging"
 
 require_relative "floe/runner"
+require_relative "floe/builtin_runner"
 
 require_relative "floe/validation_mixin"
 require_relative "floe/workflow_base"
@@ -20,6 +21,7 @@ require_relative "floe/workflow/choice_rule/or"
 require_relative "floe/workflow/choice_rule/and"
 require_relative "floe/workflow/choice_rule/data"
 require_relative "floe/workflow/context"
+require_relative "floe/workflow/item_batcher"
 require_relative "floe/workflow/item_processor"
 require_relative "floe/workflow/intrinsic_function"
 require_relative "floe/workflow/intrinsic_function/parser"

data/renovate.json

@@ -1,6 +1,5 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
-  "extends": [
-    "config:base"
-  ]
+  "inheritConfig": true,
+  "inheritConfigRepoName": "manageiq/renovate-config"
 }

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.17.0
 platform: ruby
 authors:
 - ManageIQ Developers
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-11-21 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -94,6 +93,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faraday-follow_redirects
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: manageiq-style
   requirement: !ruby/object:Gem::Requirement
@@ -165,7 +206,6 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: Floe is a runner for Amazon States Language workflows.
-email:
 executables:
 - floe
 extensions: []
@@ -182,6 +222,8 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- examples/everything.asl
+- examples/http.asl
 - examples/map.asl
 - examples/parallel.asl
 - examples/set-credential.asl
@@ -189,6 +231,9 @@ files:
 - exe/floe
 - floe.gemspec
 - lib/floe.rb
+- lib/floe/builtin_runner.rb
+- lib/floe/builtin_runner/methods.rb
+- lib/floe/builtin_runner/runner.rb
 - lib/floe/cli.rb
 - lib/floe/container_runner.rb
 - lib/floe/container_runner/docker.rb
@@ -213,6 +258,7 @@ files:
 - lib/floe/workflow/intrinsic_function.rb
 - lib/floe/workflow/intrinsic_function/parser.rb
 - lib/floe/workflow/intrinsic_function/transformer.rb
+- lib/floe/workflow/item_batcher.rb
 - lib/floe/workflow/item_processor.rb
 - lib/floe/workflow/path.rb
 - lib/floe/workflow/payload_template.rb
@@ -243,7 +289,6 @@ metadata:
   homepage_uri: https://github.com/ManageIQ/floe
   source_code_uri: https://github.com/ManageIQ/floe
   changelog_uri: https://github.com/ManageIQ/floe/blob/master/CHANGELOG.md
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -251,15 +296,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.7.0
+      version: 3.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.20
-signing_key:
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Floe is a runner for Amazon States Language workflows.
 test_files: []