floe 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f2791bb2351fad37f210264d51e059b780937e42a5cb1ab2c9c157ac01e18eea
-  data.tar.gz: 37436eb4095fc92c4ec4d07e84a09fd27a5b3e794faea09d5ebd37383f886830
+  metadata.gz: e7a1c4e51ef3ec3052e8c3c150070464ed1a8b463a3b1237774b8f1b2ff416dc
+  data.tar.gz: 8cd6fd6e8c7bed14635d906699f13dde1f5e666018f0c62709972dbddc1999cc
 SHA512:
-  metadata.gz: ef1fd3317c12e6af2cdfaaf07dc7cf9c281579d21b623b8ff1a7bcfe906794e19d4b964e3e33560a8d1a3e66abc1c7b1fe85e236c05497aaafea5dc027e8acfd
-  data.tar.gz: da683adbe9e0513613524f1e6c94380159e0cb07a34b6464c90b76b1ea9227534c1a7af6c1c884873ee9a811cbf90bc40275c4c728c20d2a8732172eab6554d6
+  metadata.gz: 73bf61a41d240922786746b0ace87c616311c4bd3f63ac59883096e53b0fce766ef791db3270a29cf257cd5315f5764d69c91bcfd4817efcadfaae72be4c226b
+  data.tar.gz: 3cb8f1ad58c7b1b7895363f46871b38b300a66eb40ebb69d2fc8ffd1b5ec303c982c0f49f89f9575b209b5e0671c2570e54670078da4e4916a3718050c9e0b9d

data/CHANGELOG.md

@@ -4,6 +4,15 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.4.0] - 2023-09-26
+### Added
+- Add ability to run workflows asynchronously ([#52](https://github.com/ManageIQ/floe/pull/92))
+- Add Workflow.wait, Workflow#step_nonblock, Workflow#step_nonblock_wait ([#92](https://github.com/ManageIQ/floe/pull/92))
+
+## [0.3.1] - 2023-08-29
+### Added
+- Add more global podman runner options ([#90])(https://github.com/ManageIQ/floe/pull/90)
+
 ## [0.3.0] - 2023-08-07
 ### Added
 - Add --network=host option to Docker/Podman runners ([#81])(https://github.com/ManageIQ/floe/pull/81)
@@ -54,7 +63,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.3.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.4.0...HEAD
+[0.4.0]: https://github.com/ManageIQ/floe/compare/v0.3.1...v0.4.0
+[0.3.1]: https://github.com/ManageIQ/floe/compare/v0.3.0...v0.3.1
 [0.3.0]: https://github.com/ManageIQ/floe/compare/v0.2.3...v0.3.0
 [0.2.3]: https://github.com/ManageIQ/floe/compare/v0.2.2...v0.2.3
 [0.2.2]: https://github.com/ManageIQ/floe/compare/v0.2.1...v0.2.2

data/Gemfile

@@ -8,4 +8,7 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
 # Specify your gem's dependencies in floe.gemspec
 gemspec
 
+gem "manageiq-style"
 gem "rake", "~> 13.0"
+gem "rspec"
+gem "rubocop"

data/README.md

@@ -56,7 +56,7 @@ bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"roleArn":
 ```ruby
 require 'floe'
 
-workflow = Floe::Workflow.load(File.read("workflow.asl"))
+workflow = Floe::Workflow.load("workflow.asl")
 workflow.run!
 ```
 
@@ -68,10 +68,51 @@ Floe::Workflow::Runner.docker_runner = Floe::Workflow::Runner::Podman.new
 # Or
 Floe::Workflow::Runner.docker_runner = Floe::Workflow::Runner::Kubernetes.new("namespace" => "default", "server" => "https://k8s.example.com:6443", "token" => "my-token")
 
-workflow = Floe::Workflow.load(File.read("workflow.asl"))
+workflow = Floe::Workflow.load("workflow.asl")
 workflow.run!
 ```
 
+### Non-Blocking Workflow Execution
+
+It is also possible to step through a workflow without blocking, and any state which
+would block will return `Errno::EAGAIN`.
+
+```ruby
+require 'floe'
+
+workflow = Floe::Workflow.load("workflow.asl")
+
+# Step through the workflow while it would not block
+workflow.run_nonblock
+
+# Go off and do some other task
+
+# Continue stepping until the workflow is finished
+workflow.run_nonblock
+```
+
+You can also use the `Floe::Workflow.wait` class method to wait on multiple workflows
+and return all that are ready to be stepped through.
+
+```ruby
+require 'floe'
+
+workflow1 = Floe::Workflow.load("workflow1.asl")
+workflow2 = Floe::Workflow.load("workflow2.asl")
+
+running_workflows = [workflow1, workflow2]
+until running_workflows.empty?
+  # Wait for any of the running workflows to be ready (up to the timeout)
+  ready_workflows = Floe::Workflow.wait(running_workflows)
+  # Step through the ready workflows until they would block
+  ready_workflows.each do |workflow|
+    loop while workflow.step_nonblock == 0
+  end
+  # Remove any finished workflows from the list of running_workflows
+  running_workflows.reject!(&:end?)
+end
+```
+
 ### Docker Runner Options
 
 #### Docker
@@ -84,7 +125,20 @@ Options supported by the Docker docker runner are:
 
 Options supported by the podman docker runner are:
 
-* `network` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+* `identity=string` - path to SSH identity file, (CONTAINER_SSHKEY)
+* `log-level=string` - Log messages above specified level (trace, debug, info, warn, warning, error, fatal, panic)
+* `network=string` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+* `noout=boolean` - do not output to stdout
+* `root=string` - Path to the root directory in which data, including images, is stored
+* `runroot=string` - Path to the 'run directory' where all state information is stored
+* `runtime=string` - Path to the OCI-compatible binary used to run containers
+* `runtime-flag=stringArray` - add global flags for the container runtime
+* `storage-driver=string` - Select which storage driver is used to manage storage of images and containers
+* `storage-opt=stringArray` - Used to pass an option to the storage driver
+* `syslog=boolean` - Output logging information to syslog as well as the console
+* `tmpdir=string` - Path to the tmp directory for libpod state content
+* `transient-store=boolean` - Enable transient container storage
+* `volumepath=string` - Path to the volume directory in which volume data is stored
 
 #### Kubernetes
 

data/exe/floe

@@ -18,9 +18,6 @@ Optimist.die(:docker_runner, "must be one of #{Floe::Workflow::Runner::TYPES.joi
 require "logger"
 Floe.logger = Logger.new($stdout)
 
-context = Floe::Workflow::Context.new(input: opts[:input])
-workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
-
 runner_klass = case opts[:docker_runner]
                when "docker"
                  Floe::Workflow::Runner::Docker
@@ -34,6 +31,9 @@ runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
 
 Floe::Workflow::Runner.docker_runner = runner_klass.new(runner_options)
 
+context = Floe::Workflow::Context.new(:input => opts[:input])
+workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
+
 workflow.run!
 
 puts workflow.output.inspect

data/floe.gemspec

@@ -34,8 +34,4 @@ Gem::Specification.new do |spec|
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "more_core_extensions"
   spec.add_dependency "optimist", "~>3.0"
-
-  spec.add_development_dependency "manageiq-style"
-  spec.add_development_dependency "rspec"
-  spec.add_development_dependency "rubocop"
 end

data/lib/floe/null_logger.rb

@@ -4,7 +4,7 @@ require 'logger'
 
 module Floe
   class NullLogger < Logger
-    def initialize(*)
+    def initialize(*) # rubocop:disable Lint/MissingSuper
     end
 
     def add(*_args, &_block)

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/floe/workflow/choice_rule/and.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class And < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          children.all? { |choice| choice.true?(context, input) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/data.rb

@@ -51,27 +51,27 @@ module Floe
           raise "No such variable [#{variable}]" if value.nil? && !%w[IsNull IsPresent].include?(compare_key)
         end
 
-        def is_null?(value)
+        def is_null?(value) # rubocop:disable Naming/PredicateName
           value.nil?
         end
 
-        def is_present?(value)
+        def is_present?(value) # rubocop:disable Naming/PredicateName
           !value.nil?
         end
 
-        def is_numeric?(value)
+        def is_numeric?(value) # rubocop:disable Naming/PredicateName
           value.kind_of?(Integer) || value.kind_of?(Float)
         end
 
-        def is_string?(value)
+        def is_string?(value) # rubocop:disable Naming/PredicateName
           value.kind_of?(String)
         end
 
-        def is_boolean?(value)
+        def is_boolean?(value) # rubocop:disable Naming/PredicateName
           [true, false].include?(value)
         end
 
-        def is_timestamp?(value)
+        def is_timestamp?(value) # rubocop:disable Naming/PredicateName
           require "date"
 
           DateTime.rfc3339(value)

data/lib/floe/workflow/choice_rule/not.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class Not < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          choice = children.first
+          !choice.true?(context, input)
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule/or.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class ChoiceRule
+      class Or < Floe::Workflow::ChoiceRule
+        def true?(context, input)
+          children.any? { |choice| choice.true?(context, input) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/choice_rule.rb

@@ -4,24 +4,28 @@ module Floe
   class Workflow
     class ChoiceRule
       class << self
-        def true?(payload, context, input)
-          build(payload).true?(context, input)
-        end
-
         def build(payload)
-          data_expression = (payload.keys & %w[And Not Or]).empty?
-          if data_expression
-            Floe::Workflow::ChoiceRule::Data.new(payload)
+          if (sub_payloads = payload["Not"])
+            Floe::Workflow::ChoiceRule::Not.new(payload, build_children([sub_payloads]))
+          elsif (sub_payloads = payload["And"])
+            Floe::Workflow::ChoiceRule::And.new(payload, build_children(sub_payloads))
+          elsif (sub_payloads = payload["Or"])
+            Floe::Workflow::ChoiceRule::Or.new(payload, build_children(sub_payloads))
           else
-            Floe::Workflow::ChoiceRule::Boolean.new(payload)
+            Floe::Workflow::ChoiceRule::Data.new(payload)
           end
         end
+
+        def build_children(sub_payloads)
+          sub_payloads.map { |payload| build(payload) }
+        end
       end
 
-      attr_reader :next, :payload, :variable
+      attr_reader :next, :payload, :variable, :children
 
-      def initialize(payload)
-        @payload = payload
+      def initialize(payload, children = nil)
+        @payload   = payload
+        @children  = children
 
         @next     = payload["Next"]
         @variable = payload["Variable"]
@@ -34,7 +38,7 @@ module Floe
       private
 
       def variable_value(context, input)
-        @variable_value ||= Path.value(variable, context, input)
+        Path.value(variable, context, input)
       end
     end
   end

data/lib/floe/workflow/context.rb

@@ -11,7 +11,7 @@ module Floe
             "Input" => input
           },
           "State"        => {},
-          "States"       => [],
+          "StateHistory" => [],
           "StateMachine" => {},
           "Task"         => {}
         }
@@ -21,16 +21,64 @@ module Floe
         @context["Execution"]
       end
 
+      def started?
+        execution.key?("StartTime")
+      end
+
+      def running?
+        started? && !ended?
+      end
+
+      def ended?
+        execution.key?("EndTime")
+      end
+
       def state
         @context["State"]
       end
 
+      def input
+        state["Input"]
+      end
+
+      def output
+        state["Output"]
+      end
+
+      def output=(val)
+        state["Output"] = val
+      end
+
+      def state_name
+        state["Name"]
+      end
+
+      def next_state
+        state["NextState"]
+      end
+
+      def next_state=(val)
+        state["NextState"] = val
+      end
+
+      def status
+        if !started?
+          "pending"
+        elsif running?
+          "running"
+        elsif state["Error"]
+          "failure"
+        else
+          "success"
+        end
+      end
+
       def state=(val)
         @context["State"] = val
       end
 
-      def states
-        @context["States"]
+      def state_history
+        @context["StateHistory"]
       end
 
       def state_machine

data/lib/floe/workflow/payload_template.rb

@@ -4,35 +4,76 @@ module Floe
   class Workflow
     class PayloadTemplate
       def initialize(payload)
-        @payload = payload
+        @payload_template = parse_payload(payload)
       end
 
       def value(context, inputs = {})
-        interpolate_value_nested(payload, context, inputs)
+        interpolate_value(payload_template, context, inputs)
       end
 
       private
 
-      attr_reader :payload
+      attr_reader :payload_template
 
-      def interpolate_value_nested(value, context, inputs)
+      def parse_payload(value)
         case value
-        when Array
-          value.map { |val| interpolate_value_nested(val, context, inputs) }
-        when Hash
-          value.to_h do |key, val|
-            if key.end_with?(".$")
-              [key.chomp(".$"), interpolate_value_nested(val, context, inputs)]
-            else
-              [key, val]
-            end
+        when Array  then parse_payload_array(value)
+        when Hash   then parse_payload_hash(value)
+        when String then parse_payload_string(value)
+        else
+          value
+        end
+      end
+
+      def parse_payload_array(value)
+        value.map { |val| parse_payload(val) }
+      end
+
+      def parse_payload_hash(value)
+        value.to_h do |key, val|
+          if key.end_with?(".$")
+            check_key_conflicts(key, value)
+
+            [key, parse_payload(val)]
+          else
+            [key, val]
           end
-        when String
-          value.start_with?("$") ? Path.value(value, context, inputs) : value
+        end
+      end
+
+      def parse_payload_string(value)
+        value.start_with?("$") ? Path.new(value) : value
+      end
+
+      def interpolate_value(value, context, inputs)
+        case value
+        when Array then interpolate_value_array(value, context, inputs)
+        when Hash  then interpolate_value_hash(value, context, inputs)
+        when Path  then value.value(context, inputs)
         else
           value
         end
       end
+
+      def interpolate_value_array(value, context, inputs)
+        value.map { |val| interpolate_value(val, context, inputs) }
+      end
+
+      def interpolate_value_hash(value, context, inputs)
+        value.to_h do |key, val|
+          if key.end_with?(".$")
+            [key.chomp(".$"), interpolate_value(val, context, inputs)]
+          else
+            [key, val]
+          end
+        end
+      end
+
+      def check_key_conflicts(key, value)
+        if value.key?(key.chomp(".$"))
+          raise Floe::InvalidWorkflowError, "both #{key} and #{key.chomp(".$")} present"
+        end
+      end
     end
   end
 end

data/lib/floe/workflow/runner/docker.rb

@@ -18,34 +18,103 @@ module Floe
 
           image = resource.sub("docker://", "")
 
-          params  = ["run", :rm]
-          params += [[:net, "host"]] if network == "host"
-          params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
-
           secrets_file = nil
+          if secrets && !secrets.empty?
+            secrets_file = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets"
+          end
+
+          output = run_container(image, env, secrets_file)
+
+          {"exit_code" => 0, "output" => output}
+        ensure
+          cleanup({"secrets_ref" => secrets_file})
+        end
+
+        def run_async!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image = resource.sub("docker://", "")
+
+          runner_context = {}
 
           if secrets && !secrets.empty?
-            secrets_file = Tempfile.new
-            secrets_file.write(secrets.to_json)
-            secrets_file.flush
+            runner_context["secrets_ref"] = create_secret(secrets)
+            env["_CREDENTIALS"] = "/run/secrets"
+          end
 
-            params << [:e, "SECRETS=/run/secrets"]
-            params << [:v, "#{secrets_file.path}:/run/secrets:z"]
+          begin
+            runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"], :detached => true)
+          rescue
+            cleanup(runner_context)
+            raise
           end
 
-          params << image
+          runner_context
+        end
 
-          logger.debug("Running docker: #{AwesomeSpawn.build_command_line("docker", params)}")
-          result = AwesomeSpawn.run!("docker", :params => params)
+        def cleanup(runner_context)
+          container_id, secrets_file = runner_context.values_at("container_ref", "secrets_ref")
 
-          [result.exit_status, result.output]
-        ensure
-          secrets_file&.close!
+          delete_container(container_id) if container_id
+          File.unlink(secrets_file)      if secrets_file && File.exist?(secrets_file)
+        end
+
+        def status!(runner_context)
+          runner_context["container_state"] = inspect_container(runner_context["container_ref"]).first&.dig("State")
+        end
+
+        def running?(runner_context)
+          runner_context.dig("container_state", "Running")
+        end
+
+        def success?(runner_context)
+          runner_context.dig("container_state", "ExitCode") == 0
+        end
+
+        def output(runner_context)
+          output = docker!("logs", runner_context["container_ref"]).output
+          runner_context["output"] = output
         end
 
         private
 
         attr_reader :network
+
+        def run_container(image, env, secrets_file, detached: false)
+          params  = ["run"]
+          params << (detached ? :detach : :rm)
+          params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+          params << [:net, "host"] if @network == "host"
+          params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
+          params << image
+
+          logger.debug("Running docker: #{AwesomeSpawn.build_command_line("docker", params)}")
+
+          result = docker!(*params)
+          result.output
+        end
+
+        def inspect_container(container_id)
+          JSON.parse(docker!("inspect", container_id).output)
+        end
+
+        def delete_container(container_id)
+          docker!("rm", container_id)
+        rescue
+          nil
+        end
+
+        def create_secret(secrets)
+          secrets_file = Tempfile.new
+          secrets_file.write(secrets.to_json)
+          secrets_file.close
+          secrets_file.path
+        end
+
+        def docker!(*params, **kwargs)
+          AwesomeSpawn.run!("docker", :params => params, **kwargs)
+        end
       end
     end
   end