floe 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e7a1c4e51ef3ec3052e8c3c150070464ed1a8b463a3b1237774b8f1b2ff416dc
-  data.tar.gz: 8cd6fd6e8c7bed14635d906699f13dde1f5e666018f0c62709972dbddc1999cc
+  metadata.gz: 0aac9121f1d393173b2fb11f9422f9e689693318a97dada576f054d827901024
+  data.tar.gz: 8118d4632e54c6fa3beb94384be517d5f3bf494890855901ce36af7ea94ee1c2
 SHA512:
-  metadata.gz: 73bf61a41d240922786746b0ace87c616311c4bd3f63ac59883096e53b0fce766ef791db3270a29cf257cd5315f5764d69c91bcfd4817efcadfaae72be4c226b
-  data.tar.gz: 3cb8f1ad58c7b1b7895363f46871b38b300a66eb40ebb69d2fc8ffd1b5ec303c982c0f49f89f9575b209b5e0671c2570e54670078da4e4916a3718050c9e0b9d
+  metadata.gz: 3872e70c2bb41e46570bcba95ef63716cdf3ea93bed77e9654db6614f8225982e13e3a83256b97854b6aa4376799e803f2edd5066dc3e304e75e2970c55d60f2
+  data.tar.gz: fac7e8815bcff8d93e443653759a6cdf8371c78f9114e3fef133212c0a1ef49d76bb35a0f491ef5abfeb134a21dc11cbbc00a6694d1f05d0e90f5bdae3471c41

data/CHANGELOG.md

@@ -4,6 +4,27 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.5.0] - 2023-10-12
+### Added
+- For task errors, use the json on the last line ([#128](https://github.com/ManageIQ/floe/pull/128))
+- Add ability to pass task service account to kube runner ([#131](https://github.com/ManageIQ/floe/pull/131))
+
+### Fixed
+- Don't put credentials file into input ([#124](https://github.com/ManageIQ/floe/pull/124))
+- exe/floe return success status if the workflow was successful ([#129](https://github.com/ManageIQ/floe/pull/129))
+- For error output, drop trailing newline ([#126](https://github.com/ManageIQ/floe/pull/126))
+
+## [0.4.1] - 2023-10-06
+### Added
+- Add Fail#CausePath and Fail#ErrorPath ([#110](https://github.com/ManageIQ/floe/pull/110))
+- Add Task#Retrier incremental backoff and Wait#Timestamp ([#100](https://github.com/ManageIQ/floe/pull/100))
+
+### Fixed
+- Combine stdout and stderr for docker and podman runners ([#104](https://github.com/ManageIQ/floe/pull/104))
+- Don't raise an exception on task failure ([#115](https://github.com/ManageIQ/floe/pull/115))
+- Fix task output handling ([#112](https://github.com/ManageIQ/floe/pull/112))
+- Fix Context#input not JSON parsed ([#122](https://github.com/ManageIQ/floe/pull/122))
+
 ## [0.4.0] - 2023-09-26
 ### Added
 - Add ability to run workflows asynchronously ([#52](https://github.com/ManageIQ/floe/pull/92))
@@ -63,7 +84,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.4.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.4.1...HEAD
+[0.4.1]: https://github.com/ManageIQ/floe/compare/v0.4.0...v0.4.1
 [0.4.0]: https://github.com/ManageIQ/floe/compare/v0.3.1...v0.4.0
 [0.3.1]: https://github.com/ManageIQ/floe/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/ManageIQ/floe/compare/v0.2.3...v0.3.0

data/Gemfile

@@ -12,3 +12,4 @@ gem "manageiq-style"
 gem "rake", "~> 13.0"
 gem "rspec"
 gem "rubocop"
+gem "timecop"

data/examples/workflow.asl

@@ -4,7 +4,7 @@
   "States": {
     "FirstState": {
       "Type": "Task",
-      "Resource": "docker://agrare/hello-world:latest",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
       "Credentials": {
         "mysecret": "dont tell anyone"
       },
@@ -49,13 +49,13 @@
 
     "FirstMatchState": {
       "Type" : "Task",
-      "Resource": "docker://agrare/hello-world:latest",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
       "Next": "PassState"
     },
 
     "SecondMatchState": {
       "Type" : "Task",
-      "Resource": "docker://agrare/hello-world:latest",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
       "Next": "NextState"
     },
 
@@ -87,7 +87,7 @@
 
     "NextState": {
       "Type": "Task",
-      "Resource": "docker://agrare/hello-world:latest",
+      "Resource": "docker://docker.io/agrare/hello-world:latest",
       "Secrets": ["vmdb:aaa-bbb-ccc"],
       "End": true
     }

data/exe/floe

@@ -37,3 +37,4 @@ workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
 workflow.run!
 
 puts workflow.output.inspect
+exit workflow.status == "success" ? 0 : 1

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.4.0"
+  VERSION = "0.5.0".freeze
 end

data/lib/floe/workflow/catcher.rb

@@ -10,7 +10,7 @@ module Floe
 
         @error_equals = payload["ErrorEquals"]
         @next         = payload["Next"]
-        @result_path  = payload.fetch("ResultPath", "$")
+        @result_path  = ReferencePath.new(payload.fetch("ResultPath", "$"))
       end
     end
   end

data/lib/floe/workflow/context.rb

@@ -5,6 +5,7 @@ module Floe
     class Context
       def initialize(context = nil, input: {})
         context = JSON.parse(context) if context.kind_of?(String)
+        input = JSON.parse(input) if input.kind_of?(String)
 
         @context = context || {
           "Execution"    => {

data/lib/floe/workflow/runner/docker.rb

@@ -13,24 +13,6 @@ module Floe
           @network = options.fetch("network", "bridge")
         end
 
-        def run!(resource, env = {}, secrets = {})
-          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
-
-          image = resource.sub("docker://", "")
-
-          secrets_file = nil
-          if secrets && !secrets.empty?
-            secrets_file = create_secret(secrets)
-            env["_CREDENTIALS"] = "/run/secrets"
-          end
-
-          output = run_container(image, env, secrets_file)
-
-          {"exit_code" => 0, "output" => output}
-        ensure
-          cleanup({"secrets_ref" => secrets_file})
-        end
-
         def run_async!(resource, env = {}, secrets = {})
           raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
 
@@ -40,11 +22,10 @@ module Floe
 
           if secrets && !secrets.empty?
             runner_context["secrets_ref"] = create_secret(secrets)
-            env["_CREDENTIALS"] = "/run/secrets"
           end
 
           begin
-            runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"], :detached => true)
+            runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"])
           rescue
             cleanup(runner_context)
             raise
@@ -73,7 +54,7 @@ module Floe
         end
 
         def output(runner_context)
-          output = docker!("logs", runner_context["container_ref"]).output
+          output = docker!("logs", runner_context["container_ref"], :combined_output => true).output
           runner_context["output"] = output
         end
 
@@ -81,10 +62,11 @@ module Floe
 
         attr_reader :network
 
-        def run_container(image, env, secrets_file, detached: false)
+        def run_container(image, env, secrets_file)
           params  = ["run"]
-          params << (detached ? :detach : :rm)
+          params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+          params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
           params << [:net, "host"] if @network == "host"
           params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
           params << image

data/lib/floe/workflow/runner/kubernetes.rb

@@ -35,35 +35,9 @@ module Floe
 
           @namespace = options.fetch("namespace", "default")
 
-          super
-        end
-
-        def run!(resource, env = {}, secrets = {})
-          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
-
-          image  = resource.sub("docker://", "")
-          name   = pod_name(image)
-          secret = create_secret!(secrets) if secrets && !secrets.empty?
-
-          begin
-            runner_context = {"container_ref" => name}
+          @task_service_account = options["task_service_account"]
 
-            create_pod!(name, image, env, secret)
-            loop do
-              case pod_info(name).dig("status", "phase")
-              when "Pending", "Running"
-                sleep(1)
-              else # also "Succeeded"
-                runner_context["exit_code"] = 0
-                output(runner_context)
-                break
-              end
-            end
-
-            runner_context
-          ensure
-            cleanup({"container_ref" => name, "secrets_ref" => secret})
-          end
+          super
         end
 
         def run_async!(resource, env = {}, secrets = {})
@@ -148,6 +122,8 @@ module Floe
             }
           }
 
+          spec[:spec][:serviceAccountName] = @task_service_account if @task_service_account
+
           if secret
             spec[:spec][:volumes] = [
               {

data/lib/floe/workflow/runner/podman.rb

@@ -26,23 +26,6 @@ module Floe
           @volumepath      = options["volumepath"]
         end
 
-        def run!(resource, env = {}, secrets = {})
-          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
-
-          image = resource.sub("docker://", "")
-
-          if secrets && !secrets.empty?
-            secret = create_secret(secrets)
-            env["_CREDENTIALS"] = "/run/secrets/#{secret}"
-          end
-
-          output = run_container(image, env, secret)
-
-          {"exit_code" => 0, :output => output}
-        ensure
-          delete_secret(secret) if secret
-        end
-
         def run_async!(resource, env = {}, secrets = {})
           raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
 
@@ -50,11 +33,10 @@ module Floe
 
           if secrets && !secrets.empty?
             secret_guid = create_secret(secrets)
-            env["_CREDENTIALS"] = "/run/secrets/#{secret_guid}"
           end
 
           begin
-            container_id = run_container(image, env, secret_guid, :detached => true)
+            container_id = run_container(image, env, secret_guid)
           rescue
             cleanup({"container_ref" => container_id, "secrets_ref" => secret_guid})
             raise
@@ -83,16 +65,17 @@ module Floe
         end
 
         def output(runner_context)
-          output = podman!("logs", runner_context["container_ref"]).output
+          output = podman!("logs", runner_context["container_ref"], :combined_output => true).output
           runner_context["output"] = output
         end
 
         private
 
-        def run_container(image, env, secret, detached: false)
+        def run_container(image, env, secret)
           params  = ["run"]
-          params << (detached ? :detach : :rm)
+          params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+          params << [:e, "_CREDENTIALS=/run/secrets/#{secret}"] if secret
           params << [:net, "host"] if @network == "host"
           params << [:secret, secret] if secret
           params << image

data/lib/floe/workflow/runner.rb

@@ -5,7 +5,8 @@ module Floe
     class Runner
       include Logging
 
-      TYPES = %w[docker podman kubernetes].freeze
+      TYPES         = %w[docker podman kubernetes].freeze
+      OUTPUT_MARKER = "__FLOE_OUTPUT__\n"
 
       def initialize(_options = {})
       end
@@ -34,23 +35,24 @@ module Floe
         raise NotImplementedError, "Must be implemented in a subclass"
       end
 
+      # @return [Hash] runner_context
       def run_async!(_image, _env = {}, _secrets = {})
         raise NotImplementedError, "Must be implemented in a subclass"
       end
 
-      def running?(_ref)
+      def running?(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
 
-      def success?(_ref)
+      def success?(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
 
-      def output(_ref)
+      def output(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
 
-      def cleanup(_ref, _secret)
+      def cleanup(_runner_context)
         raise NotImplementedError, "Must be implemented in a subclass"
       end
     end

data/lib/floe/workflow/state.rb

@@ -92,6 +92,23 @@ module Floe
       def finished?
         context.state.key?("FinishedTime")
       end
+
+      private
+
+      def wait(seconds: nil, time: nil)
+        context.state["WaitUntil"] =
+          if seconds
+            (Time.parse(context.state["EnteredTime"]) + seconds).iso8601
+          elsif time.kind_of?(String)
+            time
+          else
+            time.iso8601
+          end
+      end
+
+      def waiting?
+        context.state["WaitUntil"] && Time.now.utc <= Time.parse(context.state["WaitUntil"])
+      end
     end
   end
 end

data/lib/floe/workflow/states/fail.rb

@@ -9,16 +9,24 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          @cause = payload["Cause"]
-          @error = payload["Error"]
+          @cause      = payload["Cause"]
+          @error      = payload["Error"]
+          @cause_path = Path.new(payload["CausePath"]) if payload["CausePath"]
+          @error_path = Path.new(payload["ErrorPath"]) if payload["ErrorPath"]
         end
 
         def start(input)
           super
-          context.state["Error"] = error
-          context.state["Cause"] = cause
-          context.next_state     = nil
-          context.output         = input
+          context.next_state = nil
+          # TODO: support intrinsic functions here
+          # see https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-fail-state.html
+          #     https://docs.aws.amazon.com/step-functions/latest/dg/amazon-states-language-intrinsic-functions.html#asl-intrsc-func-generic
+          context.output     = {
+            "Error" => @error_path ? @error_path.value(context, input) : error,
+            "Cause" => @cause_path ? @cause_path.value(context, input) : cause
+          }.compact
+          context.state["Error"] = context.output["Error"]
+          context.state["Cause"] = context.output["Cause"]
         end
 
         def running?

data/lib/floe/workflow/states/task.rb

@@ -29,10 +29,10 @@ module Floe
 
         def start(input)
           super
-          input = input_path.value(context, input)
-          input = parameters.value(context, input) if parameters
 
+          input          = process_input(input)
           runner_context = runner.run_async!(resource, input, credentials&.value({}, workflow.credentials))
+
           context.state["RunnerContext"] = runner_context
         end
 
@@ -41,13 +41,15 @@ module Floe
         end
 
         def finish
-          results = runner.output(context.state["RunnerContext"])
+          output = runner.output(context.state["RunnerContext"])
 
           if success?
-            context.state["Output"] = process_output!(results)
+            output = parse_output(output)
+            context.state["Output"] = process_output!(output)
             context.next_state      = next_state
           else
-            retry_state!(results) || catch_error!(results)
+            error = parse_error(output)
+            retry_state!(error) || catch_error!(error) || fail_workflow!(error)
           end
 
           super
@@ -56,6 +58,8 @@ module Floe
         end
 
         def running?
+          return true if waiting?
+
           runner.status!(context.state["RunnerContext"])
           runner.running?(context.state["RunnerContext"])
         end
@@ -81,7 +85,7 @@ module Floe
         end
 
         def retry_state!(error)
-          retrier = find_retrier(error)
+          retrier = find_retrier(error["Error"]) if error
           return if retrier.nil?
 
           # If a different retrier is hit reset the context
@@ -94,16 +98,24 @@ module Floe
 
           return if context["State"]["RetryCount"] > retrier.max_attempts
 
-          # TODO: Kernel.sleep(retrier.sleep_duration(context["State"]["RetryCount"]))
+          wait(:seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
           context.next_state = context.state_name
           true
         end
 
         def catch_error!(error)
-          catcher = find_catcher(error)
-          raise error if catcher.nil?
+          catcher = find_catcher(error["Error"]) if error
+          return if catcher.nil?
 
           context.next_state = catcher.next
+          context.output     = catcher.result_path.set(context.input, error)
+          true
+        end
+
+        def fail_workflow!(error)
+          context.next_state     = nil
+          context.output         = {"Error" => error["Error"], "Cause" => error["Cause"]}.compact
+          context.state["Error"] = context.output["Error"]
         end
 
         def process_input(input)
@@ -112,17 +124,27 @@ module Floe
           input
         end
 
+        def parse_error(output)
+          return if output.nil?
+
+          JSON.parse(output.split("\n").last)
+        rescue JSON::ParserError
+          {"Error" => output.chomp}
+        end
+
+        def parse_output(output)
+          return if output.nil?
+
+          JSON.parse(output.split("\n").last)
+        rescue JSON::ParserError
+          nil
+        end
+
         def process_output!(results)
-          output = process_input(context.state["Input"])
+          output = context.input.dup
           return output if results.nil?
           return if output_path.nil?
 
-          begin
-            results = JSON.parse(results)
-          rescue JSON::ParserError
-            results = {"results" => results}
-          end
-
           results = result_selector.value(context, results) if result_selector
           output  = result_path.set(output, results)
           output_path.value(context, output)

data/lib/floe/workflow/states/wait.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'time'
+
 module Floe
   class Workflow
     module States
@@ -9,9 +11,12 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          @next    = payload["Next"]
-          @end     = !!payload["End"]
-          @seconds = payload["Seconds"].to_i
+          @next           = payload["Next"]
+          @end            = !!payload["End"]
+          @seconds        = payload["Seconds"]&.to_i
+          @timestamp      = payload["Timestamp"]
+          @timestamp_path = Path.new(payload["TimestampPath"]) if payload.key?("TimestampPath")
+          @seconds_path   = Path.new(payload["SecondsPath"]) if payload.key?("SecondsPath")
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
@@ -23,21 +28,25 @@ module Floe
 
           context.output     = output_path.value(context, input)
           context.next_state = end? ? nil : @next
+          please_hold(input)
         end
 
         def running?
-          now = Time.now.utc
-          if now > (Time.parse(context.state["EnteredTime"]) + @seconds)
-            context.state["FinishedTime"] = now.iso8601
-            false
-          else
-            true
-          end
+          waiting?
         end
 
         def end?
           @end
         end
+
+        private
+
+        def please_hold(input)
+          wait(
+            :seconds => @seconds_path ? @seconds_path.value(context, input).to_i : @seconds,
+            :time    => @timestamp_path ? @timestamp_path.value(context, input) : @timestamp
+          )
+        end
       end
     end
   end

data/lib/floe.rb

@@ -32,6 +32,7 @@ require_relative "floe/workflow/states/task"
 require_relative "floe/workflow/states/wait"
 
 require "jsonpath"
+require "time"
 
 module Floe
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - ManageIQ Developers
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-09-26 00:00:00.000000000 Z
+date: 2023-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_spawn
@@ -152,7 +152,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.15
+rubygems_version: 3.2.33
 signing_key:
 specification_version: 4
 summary: Simple Workflow Runner.