floe 0.2.3 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68fc7de6c5112f8717f2941c4f89f22d688092a2ab15a6fe5b859c0f6684c177
-  data.tar.gz: 23172e12b50792acbc0891dd9b0fa0a9e4d0583aa6e52e85fae408faa33ffdd4
+  metadata.gz: ae01d21cee834a23c32b89a8a1418d5c02e1a7ec825305cce2ea783c9be6fc40
+  data.tar.gz: df5f35ef07d199e929e7cf3bb0a73620a6ee5def9b555abf1409a9f74c6f7b9a
 SHA512:
-  metadata.gz: 157482ec4fd0914d18cb87c06d1ee61f91668baf9c6592451f6c8c1e5de206f088256ef8c45ff29e6d26fb175e3750e1b0ed1008a76910f6520bd424977a0763
-  data.tar.gz: 7c5c586d0087914944e577a6af4f8ca99ebc8ef15798fcb6ed5f50b46a99e368c8315b087c1ad823e2809b24e8539f479f26699b56618324b4850e98bed9b63e
+  metadata.gz: e4dcf6ccfcf37b0cd187f4dfd876d60deec4f3a1037c20f76e4b759ae61cd394f68afcb23064c3a6c30f76f56e59ff038931e1630a5e7b01393702caf339d3fa
+  data.tar.gz: 0c588d1eb15f71e4ccd9870df09d72e0e1914f6d31d51b0275f39fb9a7fdd68930eb54f254b4dfa13cc184d0cf9b7babae671f7b2bfa5df1f87d77661b3e5036

data/CHANGELOG.md

@@ -4,6 +4,18 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.3.1] - 2023-08-29
+### Added
+- Add more global podman runner options ([#90])(https://github.com/ManageIQ/floe/pull/90)
+
+## [0.3.0] - 2023-08-07
+### Added
+- Add --network=host option to Docker/Podman runners ([#81])(https://github.com/ManageIQ/floe/pull/81)
+
+### Fixed
+- Fix PayloadTemplate value transformation rules ([#78])(https://github.com/ManageIQ/floe/pull/78)
+- Move end out of the root state node ([#80])(https://github.com/ManageIQ/floe/pull/80)
+
 ## [0.2.3] - 2023-07-28
 ### Fixed
 - Fix storing next_state in Context ([#76])(https://github.com/ManageIQ/floe/pull/76)
@@ -46,7 +58,10 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.2.2...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.3.1...HEAD
+[0.3.1]: https://github.com/ManageIQ/floe/compare/v0.3.0...v0.3.1
+[0.3.0]: https://github.com/ManageIQ/floe/compare/v0.2.3...v0.3.0
+[0.2.3]: https://github.com/ManageIQ/floe/compare/v0.2.2...v0.2.3
 [0.2.2]: https://github.com/ManageIQ/floe/compare/v0.2.1...v0.2.2
 [0.2.1]: https://github.com/ManageIQ/floe/compare/v0.2.0...v0.2.1
 [0.2.0]: https://github.com/ManageIQ/floe/compare/v0.1.1...v0.2.0

data/README.md

@@ -36,6 +36,21 @@ bundle exec ruby exe/floe --workflow examples/workflow.asl --inputs='{"foo": 1}'
 bundle exec ruby exe/floe --workflow examples/workflow.asl --inputs='{"foo": 1}' --docker-runner kubernetes --docker-runner-options namespace=default server=https://k8s.example.com:6443 token=my-token
 ```
 
+If your workflow has `Credentials` you can provide a payload that will help resolve those credentials references at runtime.
+
+For example if your workflow had the following Credentials field with a JSON Path property:
+```json
+"Credentials": {
+  "RoleArn.$": "$.roleArn"
+}
+```
+
+You can provide that at runtime via the `--credentials` parameter:
+
+```
+bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"roleArn": "arn:aws:iam::111122223333:role/LambdaRole"}'
+```
+
 ### Ruby Library
 
 ```ruby
@@ -59,6 +74,31 @@ workflow.run!
 
 ### Docker Runner Options
 
+#### Docker
+
+Options supported by the Docker docker runner are:
+
+* `network` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+
+#### Podman
+
+Options supported by the podman docker runner are:
+
+* `identity=string` - path to SSH identity file, (CONTAINER_SSHKEY)
+* `log-level=string` - Log messages above specified level (trace, debug, info, warn, warning, error, fatal, panic)
+* `network=string` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+* `noout=boolean` - do not output to stdout
+* `root=string` - Path to the root directory in which data, including images, is stored
+* `runroot=string` - Path to the 'run directory' where all state information is stored
+* `runtime=string` - Path to the OCI-compatible binary used to run containers
+* `runtime-flag=stringArray` - add global flags for the container runtime
+* `storage-driver=string` - Select which storage driver is used to manage storage of images and containers
+* `storage-opt=stringArray` - Used to pass an option to the storage driver
+* `syslog=boolean` - Output logging information to syslog as well as the console
+* `tmpdir=string` - Path to the tmp directory for libpod state content
+* `transient-store=boolean` - Enable transient container storage
+* `volumepath=string` - Path to the volume directory in which volume data is stored
+
 #### Kubernetes
 
 Options supported by the kubernetes docker runner are:

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.2.3"
+  VERSION = "0.3.1"
 end

data/lib/floe/workflow/choice_rule/and.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class And < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          children.all? { |choice| choice.true?(context, input) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/not.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class Not < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          choice = children.first
+          !choice.true?(context, input)
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/or.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class Or < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          children.any? { |choice| choice.true?(context, input) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule.rb

@@ -4,24 +4,28 @@ module Floe
   class Workflow
     class ChoiceRule
       class << self
-        def true?(payload, context, input)
-          build(payload).true?(context, input)
-        end
-
         def build(payload)
-          data_expression = (payload.keys & %w[And Not Or]).empty?
-          if data_expression
-            Floe::Workflow::ChoiceRule::Data.new(payload)
+          if (sub_payloads = payload["Not"])
+            Floe::Workflow::ChoiceRule::Not.new(payload, build_children([sub_payloads]))
+          elsif (sub_payloads = payload["And"])
+            Floe::Workflow::ChoiceRule::And.new(payload, build_children(sub_payloads))
+          elsif (sub_payloads = payload["Or"])
+            Floe::Workflow::ChoiceRule::Or.new(payload, build_children(sub_payloads))
           else
-            Floe::Workflow::ChoiceRule::Boolean.new(payload)
+            Floe::Workflow::ChoiceRule::Data.new(payload)
           end
         end
+
+        def build_children(sub_payloads)
+          sub_payloads.map { |payload| build(payload) }
+        end
       end
 
-      attr_reader :next, :payload, :variable
+      attr_reader :next, :payload, :variable, :children
 
-      def initialize(payload)
-        @payload = payload
+      def initialize(payload, children = nil)
+        @payload   = payload
+        @children  = children
 
         @next     = payload["Next"]
         @variable = payload["Variable"]
@@ -34,7 +38,7 @@ module Floe
       private
 
       def variable_value(context, input)
-        @variable_value ||= Path.value(variable, context, input)
+        Path.value(variable, context, input)
       end
     end
   end

data/lib/floe/workflow/context.rb

@@ -11,7 +11,7 @@ module Floe
             "Input" => input
           },
           "State"        => {},
-          "States"       => [],
+          "StateHistory" => [],
           "StateMachine" => {},
           "Task"         => {}
         }
@@ -29,8 +29,8 @@ module Floe
         @context["State"] = val
       end
 
-      def states
-        @context["States"]
+      def state_history
+        @context["StateHistory"]
       end
 
       def state_machine

data/lib/floe/workflow/payload_template.rb

@@ -4,34 +4,76 @@ module Floe
   class Workflow
     class PayloadTemplate
       def initialize(payload)
-        @payload = payload
+        @payload_template = parse_payload(payload)
       end
 
       def value(context, inputs = {})
-        interpolate_value_nested(payload, context, inputs)
+        interpolate_value(payload_template, context, inputs)
       end
 
       private
 
-      attr_reader :payload
+      attr_reader :payload_template
 
-      def interpolate_value_nested(value, context, inputs)
+      def parse_payload(value)
         case value
-        when Array
-          value.map { |val| interpolate_value_nested(val, context, inputs) }
-        when Hash
-          value.to_h do |key, val|
-            val = interpolate_value_nested(val, context, inputs)
-            key = key.gsub(/\.\$$/, "") if key.end_with?(".$")
+        when Array  then parse_payload_array(value)
+        when Hash   then parse_payload_hash(value)
+        when String then parse_payload_string(value)
+        else
+          value
+        end
+      end
+
+      def parse_payload_array(value)
+        value.map { |val| parse_payload(val) }
+      end
 
+      def parse_payload_hash(value)
+        value.to_h do |key, val|
+          if key.end_with?(".$")
+            check_key_conflicts(key, value)
+
+            [key, parse_payload(val)]
+          else
             [key, val]
           end
-        when String
-          value.start_with?("$") ? Path.value(value, context, inputs) : value
+        end
+      end
+
+      def parse_payload_string(value)
+        value.start_with?("$") ? Path.new(value) : value
+      end
+
+      def interpolate_value(value, context, inputs)
+        case value
+        when Array then interpolate_value_array(value, context, inputs)
+        when Hash  then interpolate_value_hash(value, context, inputs)
+        when Path  then value.value(context, inputs)
         else
           value
         end
       end
+
+      def interpolate_value_array(value, context, inputs)
+        value.map { |val| interpolate_value(val, context, inputs) }
+      end
+
+      def interpolate_value_hash(value, context, inputs)
+        value.to_h do |key, val|
+          if key.end_with?(".$")
+            [key.chomp(".$"), interpolate_value(val, context, inputs)]
+          else
+            [key, val]
+          end
+        end
+      end
+
+      def check_key_conflicts(key, value)
+        if value.key?(key.chomp(".$"))
+          raise Floe::InvalidWorkflowError, "both #{key} and #{key.chomp(".$")} present"
+        end
+      end
     end
   end
 end

data/lib/floe/workflow/runner/docker.rb

@@ -4,11 +4,13 @@ module Floe
   class Workflow
     class Runner
       class Docker < Floe::Workflow::Runner
-        def initialize(*)
+        def initialize(options = {})
           require "awesome_spawn"
           require "tempfile"
 
           super
+
+          @network = options.fetch("network", "bridge")
         end
 
         def run!(resource, env = {}, secrets = {})
@@ -16,7 +18,8 @@ module Floe
 
           image = resource.sub("docker://", "")
 
-          params = ["run", :rm, [:net, "host"]]
+          params  = ["run", :rm]
+          params += [[:net, "host"]] if network == "host"
           params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
 
           secrets_file = nil
@@ -26,7 +29,7 @@ module Floe
             secrets_file.write(secrets.to_json)
             secrets_file.flush
 
-            params << [:e, "SECRETS=/run/secrets"]
+            params << [:e, "_CREDENTIALS=/run/secrets"]
             params << [:v, "#{secrets_file.path}:/run/secrets:z"]
           end
 
@@ -39,6 +42,10 @@ module Floe
         ensure
           secrets_file&.close!
         end
+
+        private
+
+        attr_reader :network
       end
     end
   end

data/lib/floe/workflow/runner/kubernetes.rb

@@ -47,27 +47,21 @@ module Floe
 
           begin
             create_pod!(name, image, env, secret)
-            while running?(name)
-              sleep(1)
+            loop do
+              case pod_info(name).dig("status", "phase")
+              when "Pending", "Running"
+                sleep(1)
+              when "Succeeded"
+                return [0, output(name)]
+              else
+                return [1, output(name)]
+              end
             end
-
-            exit_status = success?(name) ? 0 : 1
-            results     = output(name)
-
-            [exit_status, results]
           ensure
             cleanup(name, secret)
           end
         end
 
-        def running?(pod_name)
-          %w[Pending Running].include?(pod_info(pod_name).dig("status", "phase"))
-        end
-
-        def success?(pod_name)
-          pod_info(pod_name).dig("status", "phase") == "Succeeded"
-        end
-
         def output(pod)
           kubeclient.get_pod_log(pod, namespace).body
         end
@@ -125,7 +119,7 @@ module Floe
             ]
 
             spec[:spec][:containers][0][:env] << {
-              :name  => "SECRETS",
+              :name  => "_CREDENTIALS",
               :value => "/run/secrets/#{secret}/secret"
             }
 

data/lib/floe/workflow/runner/podman.rb

@@ -4,11 +4,26 @@ module Floe
   class Workflow
     class Runner
       class Podman < Floe::Workflow::Runner
-        def initialize(*)
+        def initialize(options = {})
           require "awesome_spawn"
           require "securerandom"
 
           super
+
+          @identity        = options["identity"]
+          @log_level       = options["log-level"]
+          @network         = options["network"]
+          @noout           = options["noout"].to_s == "true" if options.key?("noout")
+          @root            = options["root"]
+          @runroot         = options["runroot"]
+          @runtime         = options["runtime"]
+          @runtime_flag    = options["runtime-flag"]
+          @storage_driver  = options["storage-driver"]
+          @storage_opt     = options["storage-opt"]
+          @syslog          = options["syslog"].to_s == "true" if options.key?("syslog")
+          @tmpdir          = options["tmpdir"]
+          @transient_store = !!options["transient-store"] if options.key?("transient-store")
+          @volumepath      = options["volumepath"]
         end
 
         def run!(resource, env = {}, secrets = {})
@@ -16,26 +31,53 @@ module Floe
 
           image = resource.sub("docker://", "")
 
-          params = ["run", :rm]
+          params  = ["run", :rm]
+          params += [[:net, "host"]] if @network == "host"
           params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
 
           if secrets && !secrets.empty?
             secret_guid = SecureRandom.uuid
-            AwesomeSpawn.run!("podman", :params => ["secret", "create", secret_guid, "-"], :in_data => secrets.to_json)
+            podman!("secret", "create", secret_guid, "-", :in_data => secrets.to_json)
 
-            params << [:e, "SECRETS=/run/secrets/#{secret_guid}"]
+            params << [:e, "_CREDENTIALS=/run/secrets/#{secret_guid}"]
             params << [:secret, secret_guid]
           end
 
           params << image
 
           logger.debug("Running podman: #{AwesomeSpawn.build_command_line("podman", params)}")
-          result = AwesomeSpawn.run!("podman", :params => params)
+          result = podman!(*params)
 
           [result.exit_status, result.output]
         ensure
           AwesomeSpawn.run("podman", :params => ["secret", "rm", secret_guid]) if secret_guid
         end
+
+        private
+
+        def podman!(*args, **kwargs)
+          params = podman_global_options + args
+
+          AwesomeSpawn.run!("podman", :params => params, **kwargs)
+        end
+
+        def podman_global_options
+          options = []
+          options << [:identity, @identity]                 if @identity
+          options << [:"log-level", @log_level]             if @log_level
+          options << :noout                                 if @noout
+          options << [:root, @root]                         if @root
+          options << [:runroot, @runroot]                   if @runroot
+          options << [:runtime, @runtime]                   if @runtime
+          options << [:"runtime-flag", @runtime_flag]       if @runtime_flag
+          options << [:"storage-driver", @storage_driver]   if @storage_driver
+          options << [:"storage-opt", @storage_opt]         if @storage_opt
+          options << :syslog                                if @syslog
+          options << [:tmpdir, @tmpdir]                     if @tmpdir
+          options << [:"transient-store", @transient_store] if @transient_store
+          options << [:volumepath, @volumepath]             if @volumepath
+          options
+        end
       end
     end
   end

data/lib/floe/workflow/state.rb

@@ -25,22 +25,13 @@ module Floe
         @workflow = workflow
         @name     = name
         @payload  = payload
-        @end      = !!payload["End"]
         @type     = payload["Type"]
         @comment  = payload["Comment"]
       end
 
-      def end?
-        @end
-      end
-
       def context
         workflow.context
       end
-
-      def status
-        end? ? "success" : "running"
-      end
     end
   end
 end

data/lib/floe/workflow/states/choice.rb

@@ -23,6 +23,14 @@ module Floe
 
           [next_state, output]
         end
+
+        def status
+          "running"
+        end
+
+        def end?
+          false
+        end
       end
     end
   end

data/lib/floe/workflow/states/fail.rb

@@ -11,7 +11,6 @@ module Floe
 
           @cause = payload["Cause"]
           @error = payload["Error"]
-          @end   = true
         end
 
         def run!(input)
@@ -21,6 +20,10 @@ module Floe
         def status
           "errored"
         end
+
+        def end?
+          true
+        end
       end
     end
   end

data/lib/floe/workflow/states/pass.rb

@@ -10,6 +10,7 @@ module Floe
           super
 
           @next        = payload["Next"]
+          @end         = !!payload["End"]
           @result      = payload["Result"]
 
           @parameters  = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
@@ -23,7 +24,15 @@ module Floe
           output = result_path.set(output, result) if result && result_path
           output = output_path.value(context, output)
 
-          [@next, output]
+          [@end ? nil : @next, output]
+        end
+
+        def status
+          @end ? "success" : "running"
+        end
+
+        def end?
+          @end
         end
       end
     end

data/lib/floe/workflow/states/succeed.rb

@@ -8,13 +8,19 @@ module Floe
 
         def initialize(workflow, name, payload)
           super
-
-          @end = true
         end
 
         def run!(input)
           [nil, input]
         end
+
+        def status
+          "success"
+        end
+
+        def end?
+          true
+        end
       end
     end
   end

data/lib/floe/workflow/states/task.rb

@@ -13,6 +13,7 @@ module Floe
 
           @heartbeat_seconds = payload["HeartbeatSeconds"]
           @next              = payload["Next"]
+          @end               = !!payload["End"]
           @resource          = payload["Resource"]
           @timeout_seconds   = payload["TimeoutSeconds"]
           @retry             = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
@@ -33,7 +34,7 @@ module Floe
           _exit_status, results = runner.run!(resource, input, credentials&.value({}, workflow.credentials))
 
           output = process_output!(input, results)
-          [@next, output]
+          [@end ? nil : @next, output]
         rescue => err
           retrier = self.retry.detect { |r| (r.error_equals & [err.to_s, "States.ALL"]).any? }
           retry if retry!(retrier)
@@ -44,6 +45,14 @@ module Floe
           [catcher.next, output]
         end
 
+        def status
+          @end ? "success" : "running"
+        end
+
+        def end?
+          @end
+        end
+
         private
 
         def retry!(retrier)

data/lib/floe/workflow/states/wait.rb

@@ -10,6 +10,7 @@ module Floe
           super
 
           @next    = payload["Next"]
+          @end     = !!payload["End"]
           @seconds = payload["Seconds"].to_i
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
@@ -20,7 +21,15 @@ module Floe
           input = input_path.value(context, input)
           sleep(seconds)
           output = output_path.value(context, input)
-          [@next, output]
+          [@end ? nil : @next, output]
+        end
+
+        def status
+          @end ? "success" : "running"
+        end
+
+        def end?
+          @end
         end
       end
     end

data/lib/floe/workflow.rb

@@ -55,12 +55,12 @@ module Floe
       tock = Process.clock_gettime(Process::CLOCK_MONOTONIC)
 
       context.state["FinishedTime"] = Time.now.utc
-      context.state["Duration"]     = (tock - tick) / 1_000_000.0
+      context.state["Duration"]     = tock - tick
       context.state["Output"]       = output
 
       logger.info("Running state: [#{current_state.name}] with input [#{context["Input"]}]...Complete - next state: [#{next_state}] output: [#{output}]")
 
-      context.states << context.state
+      context.state_history << context.state
 
       @status        = current_state.status
       @current_state = next_state && @states_by_name[next_state]

data/lib/floe.rb

@@ -8,7 +8,9 @@ require_relative "floe/logging"
 require_relative "floe/workflow"
 require_relative "floe/workflow/catcher"
 require_relative "floe/workflow/choice_rule"
-require_relative "floe/workflow/choice_rule/boolean"
+require_relative "floe/workflow/choice_rule/not"
+require_relative "floe/workflow/choice_rule/or"
+require_relative "floe/workflow/choice_rule/and"
 require_relative "floe/workflow/choice_rule/data"
 require_relative "floe/workflow/context"
 require_relative "floe/workflow/path"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.3.1
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-07-28 00:00:00.000000000 Z
+date: 2023-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -148,8 +148,10 @@ files:
 - lib/floe/workflow.rb
 - lib/floe/workflow/catcher.rb
 - lib/floe/workflow/choice_rule.rb
-- lib/floe/workflow/choice_rule/boolean.rb
+- lib/floe/workflow/choice_rule/and.rb
 - lib/floe/workflow/choice_rule/data.rb
+- lib/floe/workflow/choice_rule/not.rb
+- lib/floe/workflow/choice_rule/or.rb
 - lib/floe/workflow/context.rb
 - lib/floe/workflow/path.rb
 - lib/floe/workflow/payload_template.rb

data/lib/floe/workflow/choice_rule/boolean.rb

@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module Floe
-  class Workflow
-    class ChoiceRule
-      class Boolean < Floe::Workflow::ChoiceRule
-        def true?(context, input)
-          if payload.key?("Not")
-            !ChoiceRule.true?(payload["Not"], context, input)
-          elsif payload.key?("And")
-            payload["And"].all? { |choice| ChoiceRule.true?(choice, context, input) }
-          else
-            payload["Or"].any? { |choice| ChoiceRule.true?(choice, context, input) }
-          end
-        end
-      end
-    end
-  end
-end