floe 0.9.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c7a74a5297258d481fb588ae0fa6eb1b22b7ecf5c049865b77ad23d6fb135cb
-  data.tar.gz: 82f73726e293e5345d3e7fa55a0049f881f2dce6e6d46570d2352968907c04b9
+  metadata.gz: e0c91f1170b6abe30e09d32d976efb4644fc3ac5e689274ef49a471bfb53ef8c
+  data.tar.gz: 48b2b6f51342ca80771f498dec85491f3c57ed4943e26cfbbce1ef1d3ade6029
 SHA512:
-  metadata.gz: 32d58e28cd76d936f31f9af2c1091d8a7dd930e47a2197b532c02d6d48df2c82feee696072af701aeca5f2af5437040ea3bace5df622c1ad5d0e47e388884ad2
-  data.tar.gz: 1ee0628fbfde496d00fae67812ac869582b1aca754aca7cf44caa7170edc1b0f6c9100a587770d6f6bb97bc0a948dd8fe0123fd108b9ed037ad71bc62bb8e104
+  metadata.gz: 594a89242eb6e27a345b3d5f49b93c6b362d0064d62c7154ce8e554e84f6f02a669b96a815959553c3eb6239453a26cc00a2d181d5cb7b508ee219e6891a156a
+  data.tar.gz: 519e65a57c5dd1e639705d280c89cb6868a17e05f750e7d903f8a1ae7aaae38c11839bf04fc2e7d157005fd99ca302773ddfe5cf8612c79d45efafde3de71c39

data/CHANGELOG.md

@@ -4,6 +4,35 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+## [0.11.0] - 2024-05-02
+### Fixed
+- Ensure the local code is loaded in exe/floe ([#173](https://github.com/ManageIQ/floe/pull/173))
+- Fix issues with exe/floe and various combinations of workflow and input ([#174](https://github.com/ManageIQ/floe/pull/174))
+
+### Added
+- Add support for pluggable schemes ([#169](https://github.com/ManageIQ/floe/pull/169))
+
+### Changed
+- Collapse some namespaces ([#171](https://github.com/ManageIQ/floe/pull/171))
+- Pass workflow context into runner#run_async! ([#177](https://github.com/ManageIQ/floe/pull/177))
+
+### Removed
+- Remove unused run! method ([#176](https://github.com/ManageIQ/floe/pull/176))
+
+
+## [0.10.0] - 2024-04-05
+### Fixed
+- Fix rubocops ([#164](https://github.com/ManageIQ/floe/pull/164))
+- Output should contain errors ([#165](https://github.com/ManageIQ/floe/pull/165))
+
+### Added
+- Add simplecov ([#162](https://github.com/ManageIQ/floe/pull/162))
+- Add ability to pass context on the command line ([#161](https://github.com/ManageIQ/floe/pull/161))
+- Add specs for `Workflow#wait_until`, `#waiting?` ([#166](https://github.com/ManageIQ/floe/pull/166))
+
+### Changed
+- Drop non-standard Error/Cause fields ([#167](https://github.com/ManageIQ/floe/pull/167))
+
 ## [0.9.0] - 2024-02-19
 ### Changed
 - Default to wait indefinitely ([#157](https://github.com/ManageIQ/floe/pull/157))
@@ -136,7 +165,9 @@ This project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Initial release
 
-[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.9.0...HEAD
+[Unreleased]: https://github.com/ManageIQ/floe/compare/v0.11.0...HEAD
+[0.11.0]: https://github.com/ManageIQ/floe/compare/v0.10.0...v0.11.0
+[0.10.0]: https://github.com/ManageIQ/floe/compare/v0.9.0...v0.10.0
 [0.9.0]: https://github.com/ManageIQ/floe/compare/v0.8.0...v0.9.0
 [0.8.0]: https://github.com/ManageIQ/floe/compare/v0.7.0...v0.8.0
 [0.7.0]: https://github.com/ManageIQ/floe/compare/v0.6.1...v0.7.0

data/Gemfile

@@ -7,9 +7,3 @@ require File.join(Bundler::Plugin.index.load_paths("bundler-inject")[0], "bundle
 
 # Specify your gem's dependencies in floe.gemspec
 gemspec
-
-gem "manageiq-style"
-gem "rake", "~> 13.0"
-gem "rspec"
-gem "rubocop"
-gem "timecop"

data/exe/floe

@@ -1,44 +1,52 @@
 #!/usr/bin/env ruby
 # frozen_string_literal: true
 
-require "floe"
+$LOAD_PATH.unshift(File.expand_path("../lib", __dir__))
+
 require "optimist"
+require "floe"
+require "floe/container_runner"
 
 opts = Optimist.options do
   version("v#{Floe::VERSION}\n")
   usage("[options] workflow input [workflow2 input2]")
 
-  opt :workflow, "Path to your workflow json (legacy)",         :type => :string
-  opt :input, "JSON payload to input to the workflow (legacy)", :type => :string
-  opt :credentials, "JSON payload with credentials",            :type => :string
-  opt :credentials_file, "Path to a file with credentials",     :type => :string
-  opt :docker_runner, "Type of runner for docker images",       :type => :string, :short => 'r'
-  opt :docker_runner_options, "Options to pass to the runner",  :type => :strings, :short => 'o'
+  opt :workflow, "Path to your workflow json file (alternative to passing a bare workflow)", :type => :string
+  opt :input, <<~EOMSG, :type => :string
+    JSON payload of the Input to the workflow
+      If --input is passed and --workflow is not passed, will be used for all bare workflows listed.
+      If --input is not passed and --workflow is passed, defaults to '{}'.
+  EOMSG
+  opt :context, "JSON payload of the Context",              :type => :string
+  opt :credentials, "JSON payload with Credentials",        :type => :string
+  opt :credentials_file, "Path to a file with Credentials", :type => :string
 
-  opt :docker,     "Use docker to run images     (short for --docker_runner=docker)",     :type => :boolean
-  opt :podman,     "Use podman to run images     (short for --docker_runner=podman)",     :type => :boolean
-  opt :kubernetes, "Use kubernetes to run images (short for --docker_runner=kubernetes)", :type => :boolean
+  Floe::ContainerRunner.cli_options(self)
+
+  banner("")
+  banner("General options:")
 end
 
-# legacy support for --workflow
-args = ARGV.empty? ? [opts[:workflow], opts[:input]] : ARGV
-Optimist.die(:workflow, "must be specified") if args.empty?
+# Create workflow/input pairs from the various combinations of paramaters
+args =
+  if opts[:workflow_given]
+    Optimist.die("cannot specify both --workflow and bare workflows") if ARGV.any?
 
-# shortcut support
-opts[:docker_runner] ||= "docker" if opts[:docker]
-opts[:docker_runner] ||= "podman" if opts[:podman]
-opts[:docker_runner] ||= "kubernetes" if opts[:kubernetes]
+    [opts[:workflow], opts.fetch(:input, "{}")]
+  elsif opts[:input_given]
+    Optimist.die("workflow(s) must be specified") if ARGV.empty?
 
-require "logger"
-Floe.logger = Logger.new($stdout)
+    ARGV.flat_map { |w| [w, opts[:input].dup] }
+  else
+    Optimist.die("workflow/input pairs must be specified") if ARGV.empty? || (ARGV.size > 1 && ARGV.size.odd?)
 
-runner_options = opts[:docker_runner_options].to_h { |opt| opt.split("=", 2) }
+    ARGV
+  end
 
-begin
-  Floe.set_runner("docker", opts[:docker_runner], runner_options)
-rescue ArgumentError => e
-  Optimist.die(:docker_runner, e.message)
-end
+Floe::ContainerRunner.resolve_cli_options!(opts)
+
+require "logger"
+Floe.logger = Logger.new($stdout)
 
 credentials =
   if opts[:credentials_given]
@@ -49,7 +57,7 @@ credentials =
 
 workflows =
   args.each_slice(2).map do |workflow, input|
-    context = Floe::Workflow::Context.new(:input => input || opts[:input] || "{}")
+    context = Floe::Workflow::Context.new(opts[:context], :input => input)
     Floe::Workflow.load(workflow, context, credentials)
   end
 

data/floe.gemspec

@@ -34,4 +34,11 @@ Gem::Specification.new do |spec|
   spec.add_dependency "jsonpath", "~>1.1"
   spec.add_dependency "kubeclient", "~>4.7"
   spec.add_dependency "optimist", "~>3.0"
+
+  spec.add_development_dependency "manageiq-style"
+  spec.add_development_dependency "rake", "~> 13.0"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rubocop"
+  spec.add_development_dependency "simplecov", ">= 0.21.2"
+  spec.add_development_dependency "timecop"
 end

data/lib/floe/container_runner/docker.rb

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    class Docker < Floe::Runner
+      include Floe::ContainerRunner::DockerMixin
+
+      DOCKER_COMMAND = "docker"
+
+      def initialize(options = {})
+        require "awesome_spawn"
+        require "io/wait"
+        require "tempfile"
+
+        super
+
+        @network     = options.fetch("network", "bridge")
+        @pull_policy = options["pull-policy"]
+      end
+
+      def run_async!(resource, env = {}, secrets = {}, _context = {})
+        raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+        image = resource.sub("docker://", "")
+
+        runner_context = {}
+
+        if secrets && !secrets.empty?
+          runner_context["secrets_ref"] = create_secret(secrets)
+        end
+
+        begin
+          runner_context["container_ref"] = run_container(image, env, runner_context["secrets_ref"])
+          runner_context
+        rescue AwesomeSpawn::CommandResultError => err
+          cleanup(runner_context)
+          {"Error" => "States.TaskFailed", "Cause" => err.to_s}
+        end
+      end
+
+      def cleanup(runner_context)
+        container_id, secrets_file = runner_context.values_at("container_ref", "secrets_ref")
+
+        delete_container(container_id) if container_id
+        delete_secret(secrets_file)    if secrets_file
+      end
+
+      def wait(timeout: nil, events: %i[create update delete], &block)
+        until_timestamp = Time.now.utc + timeout if timeout
+
+        r, w = IO.pipe
+
+        pid = AwesomeSpawn.run_detached(
+          self.class::DOCKER_COMMAND, :err => :out, :out => w, :params => wait_params(until_timestamp)
+        )
+
+        w.close
+
+        loop do
+          readable_timeout = until_timestamp - Time.now.utc if until_timestamp
+
+          # Wait for our end of the pipe to be readable and if it didn't timeout
+          # get the events from stdout
+          next if r.wait_readable(readable_timeout).nil?
+
+          # Get all events while the pipe is readable
+          notices = []
+          while r.ready?
+            notice = r.gets
+
+            # If the process has exited `r.gets` returns `nil` and the pipe is
+            # always `ready?`
+            break if notice.nil?
+
+            event, runner_context = parse_notice(notice)
+            next if event.nil? || !events.include?(event)
+
+            notices << [event, runner_context]
+          end
+
+          # If we're given a block yield the events otherwise return them
+          if block
+            notices.each(&block)
+          else
+            # Terminate the `docker events` process before returning the events
+            sigterm(pid)
+
+            return notices
+          end
+
+          # Check that the `docker events` process is still alive
+          Process.kill(0, pid)
+        rescue Errno::ESRCH
+          # Break out of the loop if the `docker events` process has exited
+          break
+        end
+      ensure
+        r.close
+      end
+
+      def status!(runner_context)
+        return if runner_context.key?("Error")
+
+        runner_context["container_state"] = inspect_container(runner_context["container_ref"])&.dig("State")
+      end
+
+      def running?(runner_context)
+        !!runner_context.dig("container_state", "Running")
+      end
+
+      def success?(runner_context)
+        runner_context.dig("container_state", "ExitCode") == 0
+      end
+
+      def output(runner_context)
+        return runner_context.slice("Error", "Cause") if runner_context.key?("Error")
+
+        output = docker!("logs", runner_context["container_ref"], :combined_output => true).output
+        runner_context["output"] = output
+      end
+
+      private
+
+      attr_reader :network
+
+      def run_container(image, env, secrets_file)
+        params = run_container_params(image, env, secrets_file)
+
+        logger.debug("Running #{AwesomeSpawn.build_command_line(self.class::DOCKER_COMMAND, params)}")
+
+        result = docker!(*params)
+        result.output
+      end
+
+      def run_container_params(image, env, secrets_file)
+        params  = ["run"]
+        params << :detach
+        params += env.map { |k, v| [:e, "#{k}=#{v}"] }
+        params << [:e, "_CREDENTIALS=/run/secrets"] if secrets_file
+        params << [:pull, @pull_policy] if @pull_policy
+        params << [:net, "host"] if @network == "host"
+        params << [:v, "#{secrets_file}:/run/secrets:z"] if secrets_file
+        params << [:name, container_name(image)]
+        params << image
+      end
+
+      def wait_params(until_timestamp)
+        params = ["events", [:format, "{{json .}}"], [:filter, "type=container"], [:since, Time.now.utc.to_i]]
+        params << [:until, until_timestamp.to_i] if until_timestamp
+        params
+      end
+
+      def parse_notice(notice)
+        notice = JSON.parse(notice)
+
+        status  = notice["status"]
+        event   = docker_event_status_to_event(status)
+        running = event != :delete
+
+        name, exit_code = notice.dig("Actor", "Attributes")&.values_at("name", "exitCode")
+
+        runner_context = {"container_ref" => name, "container_state" => {"Running" => running, "ExitCode" => exit_code.to_i}}
+
+        [event, runner_context]
+      rescue JSON::ParserError
+        []
+      end
+
+      def docker_event_status_to_event(status)
+        case status
+        when "create"
+          :create
+        when "start"
+          :update
+        when "die", "destroy"
+          :delete
+        else
+          :unkonwn
+        end
+      end
+
+      def inspect_container(container_id)
+        JSON.parse(docker!("inspect", container_id).output).first
+      rescue
+        nil
+      end
+
+      def delete_container(container_id)
+        docker!("rm", container_id)
+      rescue
+        nil
+      end
+
+      def delete_secret(secrets_file)
+        return unless File.exist?(secrets_file)
+
+        File.unlink(secrets_file)
+      rescue
+        nil
+      end
+
+      def create_secret(secrets)
+        secrets_file = Tempfile.new
+        secrets_file.write(secrets.to_json)
+        secrets_file.close
+        secrets_file.path
+      end
+
+      def sigterm(pid)
+        Process.kill("TERM", pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def global_docker_options
+        []
+      end
+
+      def docker!(*args, **kwargs)
+        params = global_docker_options + args
+        AwesomeSpawn.run!(self.class::DOCKER_COMMAND, :params => params, **kwargs)
+      end
+    end
+  end
+end

data/lib/floe/container_runner/docker_mixin.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Floe
+  class ContainerRunner
+    module DockerMixin
+      def image_name(image)
+        image.match(%r{^(?<repository>.+/)?(?<image>.+):(?<tag>.+)$})&.named_captures&.dig("image")
+      end
+
+      # 63 is the max kubernetes pod name length
+      # -5 for the "floe-" prefix
+      # -9 for the random hex suffix and leading hyphen
+      MAX_CONTAINER_NAME_SIZE = 63 - 5 - 9
+
+      def container_name(image)
+        name = image_name(image)
+        raise ArgumentError, "Invalid docker image [#{image}]" if name.nil?
+
+        # Normalize the image name to be used in the container name.
+        # This follows RFC 1123 Label names in Kubernetes as they are the most restrictive
+        # See https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-label-names
+        # and https://github.com/kubernetes/kubernetes/blob/952a9cb0/staging/src/k8s.io/apimachinery/pkg/util/validation/validation.go#L178-L184
+        #
+        # This does not follow the leading and trailing character restriction because we will embed it
+        # below with a prefix and suffix that already conform to the RFC.
+        normalized_name = name.downcase.gsub(/[^a-z0-9-]/, "-")[0, MAX_CONTAINER_NAME_SIZE]
+
+        "floe-#{normalized_name}-#{SecureRandom.hex(4)}"
+      end
+    end
+  end
+end