floe 0.7.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 88cfa362fcb949aeaa6e9b0b799bff4bbb0e6b7158c95920545300ba063d6a0d
-  data.tar.gz: 14df3cddf7a1811e945f533b88fcacd6611566e212cacb88bebf2da0bae9fe67
+  metadata.gz: c121968c8c3f9c70edaa20c0ab3e7937206623b7c06c0cf0f300aceddcde7814
+  data.tar.gz: 943d9e77a9e8309a771e57fee5d6f70e5026a4d254b0da5072b768a99f427d1a
 SHA512:
-  metadata.gz: 90a3cdfdb241242d5b52ad403e7bbbf409476efec6d01d16f8f6f223b6f5fa3e227647da98ab2fb9466993982793be32684521ca5a268e73ab191ff7b6a025fb
-  data.tar.gz: c1194b741493a3db2f3abdf083ef2858b25af61442584c6a2001d222579c78e5057f0a7efd3c6c904e02d898c296f543c5af1889c2e4bb91c95f9197d327fa35
+  metadata.gz: 4fa820c364e8dc3266c2fc1eb29d682e72a9d2ab4d8f154aac3a88ff6d4c35e45e124fe714b80110b52e78a0ab654e25f956bc8a39f45f05f3990c0ab26d01ed
+  data.tar.gz: b62ac48c4105497c1d3dd2f2ef9d8aec60c25bcadf71046e8488175705365d0d84facde34c11f0047660b5ab2f4390f13babbe8ba2ab6becf31027f027e0b833

data/CHANGELOG.md

@@ -4,8 +4,16 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
-## [0.7.1] - 2024-01-17
+## [0.8.0] - 2024-01-17
+### Added
+- Add CLI shorthand options for docker runner ([#147](https://github.com/ManageIQ/floe/pull/147))
+- Run multiple workflows in exe/floe ([#149](https://github.com/ManageIQ/floe/pull/149))
+- Add secure options for passing credentials via command-line ([#151](https://github.com/ManageIQ/floe/pull/151))
+- Add a Docker Runner pull-policy option ([#155](https://github.com/ManageIQ/floe/pull/155))
+
 ### Fixed
+- Fix podman with empty output ([#150](https://github.com/ManageIQ/floe/pull/150))
+- Fix run_container logger saying docker when using podman ([#154](https://github.com/ManageIQ/floe/pull/154))
 - Ensure that workflow credentials is not-nil ([#156](https://github.com/ManageIQ/floe/pull/156))
 
 ## [0.7.0] - 2023-12-18
@@ -118,8 +126,8 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.7.1...HEAD
-[0.7.1]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.7.1
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.8.0...HEAD
+[0.8.0]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/ManageIQ/floe/compare/v0.6.1...v0.7.0
 [0.6.1]: https://github.com/ManageIQ/floe/compare/v0.6.0...v0.6.1
 [0.6.0]: https://github.com/ManageIQ/floe/compare/v0.5.0...v0.6.0

data/README.md

@@ -51,6 +51,16 @@ You can provide that at runtime via the `--credentials` parameter:
 bundle exec ruby exe/floe --workflow my-workflow.asl --credentials='{"roleArn": "arn:aws:iam::111122223333:role/LambdaRole"}'
 ```
 
+Or if you are running the floe command programmatically you can securely provide the credentials via a stdin pipe via `--credentials=-`:
+```
+echo '{"roleArn": "arn:aws:iam::111122223333:role/LambdaRole"}' | bundle exec ruby exe/floe --workflow my-workflow.asl --credentials -
+```
+
+Or you can pass a file path with the `--credentials-file` parameter:
+```
+bundle exec ruby exe/floe --workflow my-workflow.asl --credentials-file /tmp/20231218-80537-kj494t
+```
+
 If you need to set a credential at runtime you can do that by using the `"ResultPath": "$.Credentials"` directive, for example to user a username/password to login and get a Bearer token:
 
 ```
@@ -152,6 +162,7 @@ end
 Options supported by the Docker docker runner are:
 
 * `network` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
+* `pull-policy` - Pull image policy. The default is missing.  Allowed values: always, missing, never
 
 #### Podman
 
@@ -161,6 +172,7 @@ Options supported by the podman docker runner are:
 * `log-level=string` - Log messages above specified level (trace, debug, info, warn, warning, error, fatal, panic)
 * `network=string` - What docker to connect the container to, defaults to `"bridge"`.  If you need access to host resources for development you can pass `network=host`.
 * `noout=boolean` - do not output to stdout
+* `pull-policy=string` - Pull image policy. The default is missing. Allowed values: always, missing, never, newer
 * `root=string` - Path to the root directory in which data, including images, is stored
 * `runroot=string` - Path to the 'run directory' where all state information is stored
 * `runtime=string` - Path to the OCI-compatible binary used to run containers
@@ -179,6 +191,7 @@ Options supported by the kubernetes docker runner are:
 * `kubeconfig` - Path to a kubeconfig file, defaults to `KUBECONFIG` environment variable or `~/.kube/config`
 * `kubeconfig_context` - Context to use in the kubeconfig file, defaults to `"default"`
 * `namespace` - Namespace to use when creating kubernetes resources, defaults to `"default"`
+* `pull-policy` - Pull image policy. The default is Always.  Allowed values: IfNotPresent, Always, Never
 * `server` - A kubernetes API Server URL, overrides anything in your kubeconfig file.  If set `KUBERNETES_SERVICE_HOST` and `KUBERNETES_SERVICE_PORT` will be used
 * `token` - A bearer_token to use to authenticate to the kubernetes API, overrides anything in your kubeconfig file.  If present, `/run/secrets/kubernetes.io/serviceaccount/token` will be used
 * `ca_file` - Path to a certificate-authority file for the kubernetes API, only valid if server and token are passed.  If present `/run/secrets/kubernetes.io/serviceaccount/ca.crt` will be used

data/exe/floe

@@ -6,15 +6,31 @@ require "optimist"
 
 opts = Optimist.options do
   version("v#{Floe::VERSION}\n")
-  opt :workflow, "Path to your workflow json",                 :type => :string, :required => true
-  opt :input, "JSON payload to input to the workflow",         :default => '{}'
-  opt :credentials, "JSON payload with credentials",           :default => "{}"
-  opt :docker_runner, "Type of runner for docker images",      :default => "docker"
-  opt :docker_runner_options, "Options to pass to the runner", :type => :strings
+  usage("[options] workflow input [workflow2 input2]")
+
+  opt :workflow, "Path to your workflow json (legacy)",         :type => :string
+  opt :input, "JSON payload to input to the workflow (legacy)", :type => :string
+  opt :credentials, "JSON payload with credentials",            :type => :string
+  opt :credentials_file, "Path to a file with credentials",     :type => :string
+  opt :docker_runner, "Type of runner for docker images",       :type => :string, :short => 'r'
+  opt :docker_runner_options, "Options to pass to the runner",  :type => :strings, :short => 'o'
+
+  opt :docker,     "Use docker to run images     (short for --docker_runner=docker)",     :type => :boolean
+  opt :podman,     "Use podman to run images     (short for --docker_runner=podman)",     :type => :boolean
+  opt :kubernetes, "Use kubernetes to run images (short for --docker_runner=kubernetes)", :type => :boolean
 end
 
 Optimist.die(:docker_runner, "must be one of #{Floe::Workflow::Runner::TYPES.join(", ")}") unless Floe::Workflow::Runner::TYPES.include?(opts[:docker_runner])
 
+# legacy support for --workflow
+args = ARGV.empty? ? [opts[:workflow], opts[:input]] : ARGV
+Optimist.die(:workflow, "must be specified") if args.empty?
+
+# shortcut support
+opts[:docker_runner] ||= "docker" if opts[:docker]
+opts[:docker_runner] ||= "podman" if opts[:podman]
+opts[:docker_runner] ||= "kubernetes" if opts[:kubernetes]
+
 require "logger"
 Floe.logger = Logger.new($stdout)
 
@@ -31,10 +47,36 @@ runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
 
 Floe::Workflow::Runner.docker_runner = runner_klass.new(runner_options)
 
-context = Floe::Workflow::Context.new(:input => opts[:input])
-workflow = Floe::Workflow.load(opts[:workflow], context, opts[:credentials])
+credentials =
+  if opts[:credentials_given]
+    opts[:credentials] == "-" ? $stdin.read : opts[:credentials]
+  elsif opts[:credentials_file_given]
+    File.read(opts[:credentials_file])
+  end
+
+workflows =
+  args.each_slice(2).map do |workflow, input|
+    context = Floe::Workflow::Context.new(:input => input || opts[:input] || "{}")
+    Floe::Workflow.load(workflow, context, credentials)
+  end
+
+# run
+
+outstanding = workflows.dup
+until outstanding.empty?
+  ready = outstanding.select(&:step_nonblock_ready?)
+  ready.map(&:run_nonblock)
+  outstanding -= ready.select(&:end?)
+  sleep(1) if !outstanding.empty?
+end
+
+# display status
+
+workflows.each do |workflow|
+  puts "", "#{workflow.name}#{" (#{workflow.status})" unless workflow.context.success?}", "===" if workflows.size > 1
+  puts workflow.output.inspect
+end
 
-workflow.run!
+# exit status
 
-puts workflow.output.inspect
-exit workflow.status == "success" ? 0 : 1
+exit workflows.all? { |workflow| workflow.context.success? } ? 0 : 1

data/lib/floe/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Floe
-  VERSION = "0.7.1".freeze
+  VERSION = "0.8.0".freeze
 end

data/lib/floe/workflow/context.rb

@@ -74,6 +74,10 @@ module Floe
         end
       end
 
+      def success?
+        status == "success"
+      end
+
       def state=(val)
         @context["State"] = val
       end

data/lib/floe/workflow/runner/docker.rb

@@ -14,7 +14,8 @@ module Floe
 
           super
 
-          @network = options.fetch("network", "bridge")
+          @network     = options.fetch("network", "bridge")
+          @pull_policy = options["pull-policy"]
         end
 
         def run_async!(resource, env = {}, secrets = {})
@@ -72,7 +73,7 @@ module Floe
         def run_container(image, env, secrets_file)
           params = run_container_params(image, env, secrets_file)
 
-          logger.debug("Running #{AwesomeSpawn.build_command_line("docker", params)}")
+          logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
 
           result = docker!(*params)
           result.output
@@ -83,6 +84,7 @@ module Floe
           params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
           params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
+          params << [:pull, @pull_policy] if @pull_policy
           params << [:net, "host"] if @network == "host"
           params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
           params << [:name, container_name(image)]

data/lib/floe/workflow/runner/kubernetes.rb

@@ -40,6 +40,7 @@ module Floe
 
           @namespace = options.fetch("namespace", "default")
 
+          @pull_policy          = options["pull-policy"]
           @task_service_account = options["task_service_account"]
 
           super
@@ -143,6 +144,7 @@ module Floe
             }
           }
 
+          spec[:spec][:imagePullPolicy]    = @pull_policy          if @pull_policy
           spec[:spec][:serviceAccountName] = @task_service_account if @task_service_account
 
           if secret

data/lib/floe/workflow/runner/podman.rb

@@ -16,6 +16,7 @@ module Floe
           @log_level       = options["log-level"]
           @network         = options["network"]
           @noout           = options["noout"].to_s == "true" if options.key?("noout")
+          @pull_policy     = options["pull-policy"]
           @root            = options["root"]
           @runroot         = options["runroot"]
           @runtime         = options["runtime"]
@@ -35,7 +36,8 @@ module Floe
           params << :detach
           params += env.map { |k, v| [:e, "#{k}=#{v}"] }
           params << [:e, "_CREDENTIALS=/run/secrets/#{secret}"] if secret
-          params << [:net, "host"] if @network == "host"
+          params << [:pull, @pull_policy] if @pull_policy
+          params << [:net, "host"]        if @network == "host"
           params << [:secret, secret] if secret
           params << [:name, container_name(image)]
           params << image

data/lib/floe/workflow/state.rb

@@ -33,10 +33,6 @@ module Floe
         raise Floe::InvalidWorkflowError, "State name [#{name}] must be less than or equal to 80 characters" if name.length > 80
       end
 
-      def run!(_input = nil)
-        wait until run_nonblock! == 0
-      end
-
       def wait(timeout: 5)
         start = Time.now.utc
 

data/lib/floe/workflow/states/task.rb

@@ -137,8 +137,8 @@ module Floe
         end
 
         def parse_output(output)
-          return if output.nil?
           return output if output.kind_of?(Hash)
+          return if output.nil? || output.empty?
 
           JSON.parse(output.split("\n").last)
         rescue JSON::ParserError

data/lib/floe/workflow.rb

@@ -8,9 +8,12 @@ module Floe
     include Logging
 
     class << self
-      def load(path_or_io, context = nil, credentials = {})
+      def load(path_or_io, context = nil, credentials = {}, name = nil)
         payload = path_or_io.respond_to?(:read) ? path_or_io.read : File.read(path_or_io)
-        new(payload, context, credentials)
+        # default the name if it is a filename and none was passed in
+        name ||= path_or_io.respond_to?(:read) ? "stream" : path_or_io.split("/").last.split(".").first
+
+        new(payload, context, credentials, name)
       end
 
       def wait(workflows, timeout: 5)
@@ -31,9 +34,9 @@ module Floe
       end
     end
 
-    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at
+    attr_reader :context, :credentials, :payload, :states, :states_by_name, :start_at, :name
 
-    def initialize(payload, context = nil, credentials = {})
+    def initialize(payload, context = nil, credentials = {}, name = nil)
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
       credentials = JSON.parse(credentials) if credentials.kind_of?(String)
       context     = Context.new(context)    unless context.kind_of?(Context)
@@ -42,12 +45,13 @@ module Floe
       raise Floe::InvalidWorkflowError, "Missing field \"StartAt\"" if payload["StartAt"].nil?
       raise Floe::InvalidWorkflowError, "\"StartAt\" not in the \"States\" field" unless payload["States"].key?(payload["StartAt"])
 
+      @name        = name
       @payload     = payload
       @context     = context
       @credentials = credentials || {}
       @start_at    = payload["StartAt"]
 
-      @states         = payload["States"].to_a.map { |name, state| State.build!(self, name, state) }
+      @states         = payload["States"].to_a.map { |state_name, state| State.build!(self, state_name, state) }
       @states_by_name = @states.each_with_object({}) { |state, result| result[state.name] = state }
 
       unless context.state.key?("Name")
@@ -58,16 +62,6 @@ module Floe
       raise Floe::InvalidWorkflowError, err.message
     end
 
-    def run!
-      step until end?
-      self
-    end
-
-    def step
-      step_nonblock_wait until step_nonblock == 0
-      self
-    end
-
     def run_nonblock
       loop while step_nonblock == 0 && !end?
       self

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: floe
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.0
 platform: ruby
 authors:
 - ManageIQ Developers