floe 0.0.1 → 0.1.0

data/lib/floe/workflow/runner/kubernetes.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class Runner
+      class Kubernetes < Floe::Workflow::Runner
+        attr_reader :namespace
+
+        def initialize(*)
+          require "awesome_spawn"
+          require "securerandom"
+          require "base64"
+          require "yaml"
+
+          @namespace = ENV.fetch("DOCKER_RUNNER_NAMESPACE", "default")
+
+          super
+        end
+
+        def run!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image     = resource.sub("docker://", "")
+          name      = pod_name(image)
+          secret    = create_secret!(secrets) unless secrets&.empty?
+          overrides = pod_spec(image, env, secret)
+
+          result = kubectl_run!(image, name, overrides)
+
+          # Kubectl prints that the pod was deleted, strip this from the output
+          output = result.output.gsub(/pod \"#{name}\" deleted/, "")
+
+          [result.exit_status, output]
+        ensure
+          delete_secret!(secret) if secret
+        end
+
+        private
+
+        def container_name(image)
+          image.match(%r{^(?<repository>.+\/)?(?<image>.+):(?<tag>.+)$})&.named_captures&.dig("image")
+        end
+
+        def pod_name(image)
+          container_short_name = container_name(image)
+          raise ArgumentError, "Invalid docker image [#{image}]" if container_short_name.nil?
+
+          "#{container_short_name}-#{SecureRandom.uuid}"
+        end
+
+        def pod_spec(image, env, secret = nil)
+          container_spec = {
+            "name" => container_name(image),
+            "image" => image,
+            "env" => env.to_h.map { |k, v| {"name" => k, "value" => v} }
+          }
+
+          spec = {"spec" => {"containers" => [container_spec]}}
+
+          if secret
+            spec["spec"]["volumes"] = [{"name"   => "secret-volume", "secret" => {"secretName" => secret}}]
+            container_spec["env"] << {"name" => "SECRETS", "value" => "/run/secrets/#{secret}/secret"}
+            container_spec["volumeMounts"] = [
+              {
+                "name"      => "secret-volume",
+                "mountPath" => "/run/secrets/#{secret}",
+                "readOnly"  => true
+              }
+            ]
+          end
+
+          spec
+        end
+
+        def create_secret!(secrets)
+          secret_name = SecureRandom.uuid
+
+          secret_yaml = {
+            "kind"       => "Secret",
+            "apiVersion" => "v1",
+            "metadata"   => {
+              "name" => secret_name,
+              "namespace" => namespace
+            },
+            "data"       => {
+              "secret" => Base64.urlsafe_encode64(secrets.to_json)
+            },
+            "type"       => "Opaque"
+          }.to_yaml
+
+          kubectl!("create", "-f", "-", :in_data => secret_yaml)
+
+          secret_name
+        end
+
+        def delete_secret!(secret_name)
+          kubectl!("delete", "secret", secret_name, [:namespace, namespace])
+        end
+
+        def kubectl!(*params, **kwargs)
+          AwesomeSpawn.run!("kubectl", :params => params, **kwargs)
+        end
+
+        def kubectl_run!(image, name, overrides = nil)
+          params = [
+            "run", :rm, :attach, [:image, image], [:restart, "Never"], [:namespace, namespace], name
+          ]
+
+          params << "--overrides=#{overrides.to_json}" if overrides
+
+          logger.debug("Running kubectl: #{AwesomeSpawn.build_command_line("kubectl", params)}")
+
+          kubectl!(*params)
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/runner/podman.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class Runner
+      class Podman < Floe::Workflow::Runner
+        def initialize(*)
+          require "awesome_spawn"
+          require "securerandom"
+
+          super
+        end
+
+        def run!(resource, env = {}, secrets = {})
+          raise ArgumentError, "Invalid resource" unless resource&.start_with?("docker://")
+
+          image = resource.sub("docker://", "")
+
+          params = ["run", :rm]
+          params += env.map { |k, v| [:e, "#{k}=#{v}"] } if env
+
+          if secrets && !secrets.empty?
+            secret_guid = SecureRandom.uuid
+            AwesomeSpawn.run!("podman", :params => ["secret", "create", secret_guid, "-"], :in_data => secrets.to_json)
+
+            params << [:e, "SECRETS=/run/secrets/#{secret_guid}"]
+            params << [:secret, secret_guid]
+          end
+
+          params << image
+
+          logger.debug("Running podman: #{AwesomeSpawn.build_command_line("podman", params)}")
+          result = AwesomeSpawn.run!("podman", :params => params)
+
+          [result.exit_status, result.output]
+        ensure
+          AwesomeSpawn.run("podman", :params => ["secret", "rm", secret_guid]) if secret_guid
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/runner.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class Runner
+      include Logging
+
+      class << self
+        attr_writer :docker_runner_klass
+
+        def docker_runner_klass
+          @docker_runner_klass ||= const_get(ENV.fetch("DOCKER_RUNNER", "docker").capitalize)
+        end
+
+        def for_resource(resource)
+          raise ArgumentError, "resource cannot be nil" if resource.nil?
+
+          scheme = resource.split("://").first
+          case scheme
+          when "docker"
+            docker_runner_klass.new
+          else
+            raise "Invalid resource scheme [#{scheme}]"
+          end
+        end
+      end
+
+      def run!(image, env = {}, secrets = {})
+        raise NotImplementedError, "Must be implemented in a subclass"
+      end
+    end
+  end
+end

data/lib/floe/workflow/state.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    class State
+      include Logging
+
+      class << self
+        def build!(workflow, name, payload)
+          state_type = payload["Type"]
+
+          begin
+            klass = Floe::Workflow::States.const_get(state_type)
+          rescue NameError
+            raise Floe::InvalidWorkflowError, "Invalid state type: [#{state_type}]"
+          end
+
+          klass.new(workflow, name, payload)
+        end
+      end
+
+      attr_reader :workflow, :comment, :name, :type, :payload
+
+      def initialize(workflow, name, payload)
+        @workflow = workflow
+        @name     = name
+        @payload  = payload
+        @end      = !!payload["End"]
+        @type     = payload["Type"]
+        @comment  = payload["Comment"]
+      end
+
+      def end?
+        @end
+      end
+
+      def context
+        workflow.context
+      end
+
+      def run!(input)
+        logger.info("Running state: [#{name}] with input [#{input}]")
+
+        input = input_path.value(context, input)
+
+        output, next_state = block_given? ? yield(input) : input
+        next_state ||= workflow.states_by_name[payload["Next"]] unless end?
+
+        output ||= input
+        output   = output_path&.value(context, output)
+
+        logger.info("Running state: [#{name}] with input [#{input}]...Complete - next state: [#{next_state&.name}] output: [#{output}]")
+
+        [next_state, output]
+      end
+
+      def to_dot
+        String.new.tap do |s|
+          s << "  #{name}"
+
+          attributes = to_dot_attributes
+          s << " [ #{attributes.to_a.map { |kv| kv.join("=") }.join(" ")} ]" unless attributes.empty?
+        end
+      end
+
+      private def to_dot_attributes
+        end? ? {:style => "bold"} : {}
+      end
+
+      def to_dot_transitions
+        next_state_name = payload["Next"] unless end?
+        Array(next_state_name && "  #{name} -> #{next_state_name}")
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/choice.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Choice < Floe::Workflow::State
+        attr_reader :choices, :default, :input_path, :output_path
+
+        def initialize(workflow, name, payload)
+          super
+
+          @choices = payload["Choices"].map { |choice| ChoiceRule.build(choice) }
+          @default = payload["Default"]
+
+          @input_path  = Path.new(payload.fetch("InputPath", "$"))
+          @output_path = Path.new(payload.fetch("OutputPath", "$"))
+        end
+
+        def run!(*)
+          super do |input|
+            next_state_name = choices.detect { |choice| choice.true?(context, input) }&.next || default
+            next_state      = workflow.states_by_name[next_state_name]
+
+            output = input
+
+            [output, next_state]
+          end
+        end
+
+        private def to_dot_attributes
+          super.merge(:shape => "diamond")
+        end
+
+        def to_dot_transitions
+          [].tap do |a|
+            choices.each do |choice|
+              choice_label =
+                if choice.payload["NumericEquals"]
+                  "#{choice.variable} == #{choice.payload["NumericEquals"]}"
+                else
+                  "Unknown" # TODO
+                end
+
+              a << "  #{name} -> #{choice.next} [ label=#{choice_label.inspect} ]"
+            end
+
+            a << "  #{name} -> #{default} [ label=\"Default\" ]" if default
+          end
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/fail.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Fail < Floe::Workflow::State
+        attr_reader :cause, :error
+
+        def initialize(workflow, name, payload)
+          super
+
+          @cause = payload["Cause"]
+          @error = payload["Error"]
+        end
+
+        def run!(input)
+          logger.info("Running state: [#{name}] with input [#{input}]")
+
+          next_state = nil
+          output     = input
+
+          logger.info("Running state: [#{name}] with input [#{input}]...Complete - next state: [#{next_state&.name}]")
+
+          [next_state, output]
+        end
+
+        def end?
+          true
+        end
+
+        private def to_dot_attributes
+          super.merge(:color => "red")
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/map.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Map < Floe::Workflow::State
+        def initialize(*)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/parallel.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Parallel < Floe::Workflow::State
+        def initialize(*)
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/pass.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Pass < Floe::Workflow::State
+        attr_reader :end, :next, :result, :parameters, :input_path, :output_path, :result_path
+
+        def initialize(workflow, name, payload)
+          super
+
+          @next        = payload["Next"]
+          @result      = payload["Result"]
+
+          @parameters  = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
+          @input_path  = Path.new(payload.fetch("InputPath", "$"))
+          @output_path = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path = ReferencePath.new(payload.fetch("ResultPath", "$"))
+        end
+
+        def run!(*)
+          super do |input|
+            output = input
+            output = result_path.set(output, result) if result && result_path
+            output
+          end
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/succeed.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Succeed < Floe::Workflow::State
+        attr_reader :input_path, :output_path
+
+        def initialize(workflow, name, payload)
+          super
+
+          @input_path  = Path.new(payload.fetch("InputPath", "$"))
+          @output_path = Path.new(payload.fetch("OutputPath", "$"))
+        end
+
+        def end?
+          true # TODO: Handle if this is ending a parallel or map state
+        end
+
+        private def to_dot_attributes
+          super.merge(:color => "green")
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/task.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Task < Floe::Workflow::State
+        attr_reader :credentials, :end, :heartbeat_seconds, :next, :parameters,
+                    :result_selector, :resource, :timeout_seconds, :retry, :catch,
+                    :input_path, :output_path, :result_path
+
+        def initialize(workflow, name, payload)
+          super
+
+          @heartbeat_seconds = payload["HeartbeatSeconds"]
+          @next              = payload["Next"]
+          @resource          = payload["Resource"]
+          @timeout_seconds   = payload["TimeoutSeconds"]
+          @retry             = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
+          @catch             = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @input_path        = Path.new(payload.fetch("InputPath", "$"))
+          @output_path       = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path       = ReferencePath.new(payload.fetch("ResultPath", "$"))
+          @parameters        = PayloadTemplate.new(payload["Parameters"])     if payload["Parameters"]
+          @result_selector   = PayloadTemplate.new(payload["ResultSelector"]) if payload["ResultSelector"]
+          @credentials       = PayloadTemplate.new(payload["Credentials"])    if payload["Credentials"]
+        end
+
+        def run!(*)
+          super do |input|
+            input = parameters.value(context, input) if parameters
+
+            runner = Floe::Workflow::Runner.for_resource(resource)
+            _exit_status, results = runner.run!(resource, input, credentials&.value({}, workflow.credentials))
+
+            output = input
+            process_output!(output, results)
+          rescue => err
+            retrier = self.retry.detect { |r| (r.error_equals & [err.to_s, "States.ALL"]).any? }
+            retry if retry!(retrier)
+
+            catcher = self.catch.detect { |c| (c.error_equals & [err.to_s, "States.ALL"]).any? }
+            raise if catcher.nil?
+
+            [output, workflow.states_by_name[catcher.next]]
+          end
+        end
+
+        private
+
+        def retry!(retrier)
+          return if retrier.nil?
+
+          # If a different retrier is hit reset the context
+          if !context.key?("retrier") || context["retrier"]["error_equals"] != retrier.error_equals
+            context["retrier"] = {"error_equals" => retrier.error_equals, "retry_count" => 0}
+          end
+
+          context["retrier"]["retry_count"] += 1
+
+          return if context["retrier"]["retry_count"] > retrier.max_attempts
+
+          Kernel.sleep(retrier.sleep_duration(context["retrier"]["retry_count"]))
+          true
+        end
+
+        def process_output!(output, results)
+          return output if results.nil?
+          return if output_path.nil?
+
+          begin
+            results = JSON.parse(results)
+          rescue JSON::ParserError
+            results = {"results" => results}
+          end
+
+          results = result_selector.value(context, results) if result_selector
+          result_path.set(output, results)
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/wait.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      class Wait < Floe::Workflow::State
+        attr_reader :end, :next, :seconds, :input_path, :output_path
+
+        def initialize(workflow, name, payload)
+          super
+
+          @next    = payload["Next"]
+          @seconds = payload["Seconds"].to_i
+
+          @input_path  = Path.new(payload.fetch("InputPath", "$"), context)
+          @output_path = Path.new(payload.fetch("OutputPath", "$"), context)
+        end
+
+        def run!(*)
+          super { sleep(seconds); nil }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Floe
+  class Workflow
+    class << self
+      def load(path_or_io, context = {}, credentials = {})
+        payload = path_or_io.respond_to?(:read) ? path_or_io.read : File.read(path_or_io)
+        new(payload, context, credentials)
+      end
+    end
+
+    attr_reader :context, :credentials, :first_state, :payload, :states, :states_by_name, :start_at
+
+    def initialize(payload, context = {}, credentials = {})
+      payload     = JSON.parse(payload)     if payload.kind_of?(String)
+      context     = JSON.parse(context)     if context.kind_of?(String)
+      credentials = JSON.parse(credentials) if credentials.kind_of?(String)
+
+      @payload        = payload
+      @context        = context
+      @credentials    = credentials
+      @states         = payload["States"].to_a.map { |name, state| State.build!(self, name, state) }
+      @states_by_name = states.to_h { |state| [state.name, state] }
+      @start_at       = @payload["StartAt"]
+      @first_state    = @states_by_name[@start_at]
+    rescue JSON::ParserError => err
+      raise Floe::InvalidWorkflowError, err.message
+    end
+
+    def run!
+      state = first_state
+      input = context.dup
+
+      until state.nil?
+        state, output = state.run!(input)
+        input = output
+      end
+
+      output
+    end
+
+    def to_dot
+      String.new.tap do |s|
+        s << "digraph {\n"
+        states.each do |state|
+          s << state.to_dot << "\n"
+        end
+        s << "\n"
+        states.each do |state|
+          Array(state.to_dot_transitions).each do |transition|
+            s << transition << "\n"
+          end
+        end
+        s << "}\n"
+      end
+    end
+
+    def to_svg(path: nil)
+      require "open3"
+      out, err, _status = Open3.capture3("dot -Tsvg", :stdin_data => to_dot)
+
+      raise "Error from graphviz:\n#{err}" if err && !err.empty?
+
+      File.write(path, out) if path
+
+      out
+    end
+
+    def to_ascii(path: nil)
+      require "open3"
+      out, err, _status = Open3.capture3("graph-easy", :stdin_data => to_dot)
+
+      raise "Error from graph-easy:\n#{err}" if err && !err.empty?
+
+      File.write(path, out) if path
+
+      out
+    end
+  end
+end

data/lib/floe.rb

@@ -1,5 +1,44 @@
-require "floe/version"
+# frozen_string_literal: true
+
+require_relative "floe/version"
+
+require_relative "floe/null_logger"
+require_relative "floe/logging"
+
+require_relative "floe/workflow"
+require_relative "floe/workflow/catcher"
+require_relative "floe/workflow/choice_rule"
+require_relative "floe/workflow/choice_rule/boolean"
+require_relative "floe/workflow/choice_rule/data"
+require_relative "floe/workflow/path"
+require_relative "floe/workflow/payload_template"
+require_relative "floe/workflow/reference_path"
+require_relative "floe/workflow/retrier"
+require_relative "floe/workflow/runner"
+require_relative "floe/workflow/runner/docker"
+require_relative "floe/workflow/runner/kubernetes"
+require_relative "floe/workflow/runner/podman"
+require_relative "floe/workflow/state"
+require_relative "floe/workflow/states/choice"
+require_relative "floe/workflow/states/fail"
+require_relative "floe/workflow/states/map"
+require_relative "floe/workflow/states/parallel"
+require_relative "floe/workflow/states/pass"
+require_relative "floe/workflow/states/succeed"
+require_relative "floe/workflow/states/task"
+require_relative "floe/workflow/states/wait"
+
+require "jsonpath"
 
 module Floe
-  # Your code goes here...
+  class Error < StandardError; end
+  class InvalidWorkflowError < Error; end
+
+  def self.logger
+    @logger ||= NullLogger.new
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+  end
 end