floe 0.14.0 → 0.15.1

data/lib/floe/workflow/states/child_workflow_mixin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      module ChildWorkflowMixin
+        def run_nonblock!(context)
+          start(context) unless context.state_started?
+          step_nonblock!(context)
+          return Errno::EAGAIN unless ready?(context)
+
+          finish(context) if ended?(context)
+        end
+
+        def finish(context)
+          if success?(context)
+            result = each_child_context(context).map(&:output)
+            context.output = process_output(context, result)
+          else
+            error = parse_error(context)
+            retry_state!(context, error) || catch_error!(context, error) || fail_workflow!(context, error)
+          end
+
+          super
+        end
+
+        def ready?(context)
+          !context.state_started? || each_child_workflow(context).any? { |wf, ctx| wf.step_nonblock_ready?(ctx) }
+        end
+
+        def wait_until(context)
+          each_child_workflow(context).filter_map { |wf, ctx| wf.wait_until(ctx) }.min
+        end
+
+        def waiting?(context)
+          each_child_workflow(context).any? { |wf, ctx| wf.waiting?(ctx) }
+        end
+
+        def running?(context)
+          !ended?(context)
+        end
+
+        def ended?(context)
+          each_child_context(context).all?(&:ended?)
+        end
+
+        def success?(context)
+          each_child_context(context).none?(&:failed?)
+        end
+
+        def each_child_context(context)
+          context.state[child_context_key].map { |ctx| Context.new(ctx) }
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/choice.rb

@@ -9,10 +9,9 @@ module Floe
         def initialize(workflow, name, payload)
           super
 
-          validate_state!(workflow)
-
-          @choices = payload["Choices"].map.with_index { |choice, i| ChoiceRule.build(workflow, name + ["Choices", i.to_s], choice) }
+          @choices = payload["Choices"]&.map&.with_index { |choice, i| ChoiceRule.build(workflow, name + ["Choices", i.to_s], choice) }
           @default = payload["Default"]
+          validate_state!(workflow)
 
           @input_path  = Path.new(payload.fetch("InputPath", "$"))
           @output_path = Path.new(payload.fetch("OutputPath", "$"))
@@ -45,12 +44,12 @@ module Floe
         end
 
         def validate_state_choices!
-          missing_field_error!("Choices") unless payload.key?("Choices")
-          invalid_field_error!("Choices", nil, "must be a non-empty array") unless payload["Choices"].kind_of?(Array) && !payload["Choices"].empty?
+          missing_field_error!("Choices") if @choices.nil?
+          invalid_field_error!("Choices", nil, "must be a non-empty array") unless @choices.kind_of?(Array) && !@choices.empty?
         end
 
         def validate_state_default!(workflow)
-          invalid_field_error!("Default", payload["Default"], "is not found in \"States\"") if payload["Default"] && !workflow_state?(payload["Default"], workflow)
+          invalid_field_error!("Default", @default, "is not found in \"States\"") if @default && !workflow_state?(@default, workflow)
         end
       end
     end

data/lib/floe/workflow/states/map.rb

@@ -4,9 +4,123 @@ module Floe
   class Workflow
     module States
       class Map < Floe::Workflow::State
-        def initialize(*)
+        include ChildWorkflowMixin
+        include InputOutputMixin
+        include NonTerminalMixin
+        include RetryCatchMixin
+
+        attr_reader :end, :next, :parameters, :input_path, :output_path, :result_path,
+                    :result_selector, :retry, :catch, :item_processor, :items_path,
+                    :item_reader, :item_selector, :item_batcher, :result_writer,
+                    :max_concurrency, :tolerated_failure_percentage, :tolerated_failure_count
+
+        def initialize(workflow, name, payload)
+          super
+
+          missing_field_error!("InputProcessor") if payload["ItemProcessor"].nil?
+
+          @next            = payload["Next"]
+          @end             = !!payload["End"]
+          @parameters      = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
+          @input_path      = Path.new(payload.fetch("InputPath", "$"))
+          @output_path     = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path     = ReferencePath.new(payload.fetch("ResultPath", "$"))
+          @result_selector = PayloadTemplate.new(payload["ResultSelector"]) if payload["ResultSelector"]
+          @retry           = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
+          @catch           = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @item_processor  = ItemProcessor.new(payload["ItemProcessor"], name)
+          @items_path      = ReferencePath.new(payload.fetch("ItemsPath", "$"))
+          @item_reader     = payload["ItemReader"]
+          @item_selector   = payload["ItemSelector"]
+          @item_batcher    = payload["ItemBatcher"]
+          @result_writer   = payload["ResultWriter"]
+          @max_concurrency = payload["MaxConcurrency"]&.to_i
+          @tolerated_failure_percentage = payload["ToleratedFailurePercentage"]&.to_i
+          @tolerated_failure_count      = payload["ToleratedFailureCount"]&.to_i
+
+          validate_state!(workflow)
+        end
+
+        def process_input(context)
+          input = super
+          items_path.value(context, input)
+        end
+
+        def start(context)
           super
-          raise NotImplementedError
+
+          input = process_input(context)
+
+          context.state["ItemProcessorContext"] = input.map { |item| Context.new({"Execution" => {"Id" => context.execution["Id"]}}, :input => item.to_json).to_h }
+        end
+
+        def end?
+          @end
+        end
+
+        def success?(context)
+          contexts   = each_child_context(context)
+          num_failed = contexts.count(&:failed?)
+          total      = contexts.count
+
+          return true if num_failed.zero? || total.zero?
+          return false if tolerated_failure_count.nil? && tolerated_failure_percentage.nil?
+
+          # Some have failed, check the tolerated_failure thresholds to see if
+          # we should fail the whole state.
+          #
+          # If either ToleratedFailureCount or ToleratedFailurePercentage are breached
+          # then the whole state is considered failed.
+          count_tolerated = tolerated_failure_count.nil?      || num_failed < tolerated_failure_count
+          pct_tolerated   = tolerated_failure_percentage.nil? || tolerated_failure_percentage == 100 ||
+                            ((100 * num_failed / total.to_f) < tolerated_failure_percentage)
+
+          count_tolerated && pct_tolerated
+        end
+
+        private
+
+        def step_nonblock!(context)
+          each_child_context(context).each do |ctx|
+            # If this iteration isn't already running and we can't start any more
+            next if !ctx.started? && concurrency_exceeded?(context)
+
+            item_processor.run_nonblock(ctx) if item_processor.step_nonblock_ready?(ctx)
+          end
+        end
+
+        def each_child_workflow(context)
+          each_child_context(context).map do |ctx|
+            [item_processor, Context.new(ctx)]
+          end
+        end
+
+        def concurrency_exceeded?(context)
+          max_concurrency && num_running(context) >= max_concurrency
+        end
+
+        def num_running(context)
+          each_child_context(context).count(&:running?)
+        end
+
+        def parse_error(context)
+          # If ToleratedFailureCount or ToleratedFailurePercentage is present
+          # then use States.ExceedToleratedFailureThreshold otherwise
+          # take the error from the first failed state
+          if tolerated_failure_count || tolerated_failure_percentage
+            {"Error" => "States.ExceedToleratedFailureThreshold"}
+          else
+            each_child_context(context).detect(&:failed?)&.output || {"Error" => "States.Error"}
+          end
+        end
+
+        def child_context_key
+          "ItemProcessorContext"
+        end
+
+        def validate_state!(workflow)
+          validate_state_next!(workflow)
+          invalid_field_error!("MaxConcurrency", @max_concurrency, "must be greater than 0") if @max_concurrency && @max_concurrency <= 0
         end
       end
     end

data/lib/floe/workflow/states/parallel.rb

@@ -4,9 +4,72 @@ module Floe
   class Workflow
     module States
       class Parallel < Floe::Workflow::State
-        def initialize(*)
+        include ChildWorkflowMixin
+        include InputOutputMixin
+        include NonTerminalMixin
+        include RetryCatchMixin
+
+        attr_reader :end, :next, :parameters, :input_path, :output_path, :result_path,
+                    :result_selector, :retry, :catch, :branches
+
+        def initialize(workflow, name, payload)
+          super
+
+          missing_field_error!("Branches") if payload["Branches"].nil?
+
+          @next            = payload["Next"]
+          @end             = !!payload["End"]
+          @parameters      = PayloadTemplate.new(payload["Parameters"]) if payload["Parameters"]
+          @input_path      = Path.new(payload.fetch("InputPath", "$"))
+          @output_path     = Path.new(payload.fetch("OutputPath", "$"))
+          @result_path     = ReferencePath.new(payload.fetch("ResultPath", "$"))
+          @result_selector = PayloadTemplate.new(payload["ResultSelector"]) if payload["ResultSelector"]
+          @retry           = payload["Retry"].to_a.map { |retrier| Retrier.new(retrier) }
+          @catch           = payload["Catch"].to_a.map { |catcher| Catcher.new(catcher) }
+          @branches        = payload["Branches"].map { |branch| Branch.new(branch) }
+
+          validate_state!(workflow)
+        end
+
+        def start(context)
           super
-          raise NotImplementedError
+
+          input = process_input(context)
+
+          context.state["BranchContext"] = branches.map { |_branch| Context.new({"Execution" => {"Id" => context.execution["Id"]}}, :input => input.to_json).to_h }
+        end
+
+        def end?
+          @end
+        end
+
+        private
+
+        def step_nonblock!(context)
+          each_child_workflow(context).each do |wf, ctx|
+            wf.run_nonblock(ctx) if wf.step_nonblock_ready?(ctx)
+          end
+        end
+
+        def each_child_workflow(context)
+          branches.filter_map.with_index do |branch, i|
+            ctx = context.state.dig("BranchContext", i)
+            next if ctx.nil?
+
+            [branch, Context.new(ctx)]
+          end
+        end
+
+        def parse_error(context)
+          each_child_context(context).detect(&:failed?)&.output || {"Error" => "States.Error"}
+        end
+
+        def child_context_key
+          "BranchContext"
+        end
+
+        def validate_state!(workflow)
+          validate_state_next!(workflow)
         end
       end
     end

data/lib/floe/workflow/states/retry_catch_mixin.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Floe
+  class Workflow
+    module States
+      module RetryCatchMixin
+        def find_retrier(error)
+          self.retry.detect { |r| r.match_error?(error) }
+        end
+
+        def find_catcher(error)
+          self.catch.detect { |c| c.match_error?(error) }
+        end
+
+        def retry_state!(context, error)
+          retrier = find_retrier(error["Error"]) if error
+          return if retrier.nil?
+
+          # If a different retrier is hit reset the context
+          if !context["State"].key?("RetryCount") || context["State"]["Retrier"] != retrier.error_equals
+            context["State"]["RetryCount"] = 0
+            context["State"]["Retrier"]    = retrier.error_equals
+          end
+
+          context["State"]["RetryCount"] += 1
+
+          return if context["State"]["RetryCount"] > retrier.max_attempts
+
+          wait_until!(context, :seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
+          context.next_state = context.state_name
+          context.output     = error
+          context.logger.info("Running state: [#{long_name}] with input [#{context.json_input}] got error[#{context.json_output}]...Retry - delay: #{wait_until(context)}")
+          true
+        end
+
+        def catch_error!(context, error)
+          catcher = find_catcher(error["Error"]) if error
+          return if catcher.nil?
+
+          context.next_state = catcher.next
+          context.output     = catcher.result_path.set(context.input, error)
+          context.logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...CatchError - next state: [#{context.next_state}] output: [#{context.json_output}]")
+
+          true
+        end
+
+        def fail_workflow!(context, error)
+          # next_state is nil, and will be set to nil again in super
+          # keeping in here for completeness
+          context.next_state = nil
+          context.output = error
+          context.logger.error("Running state: [#{long_name}] with input [#{context.json_input}]...Complete workflow - output: [#{context.json_output}]")
+        end
+      end
+    end
+  end
+end

data/lib/floe/workflow/states/task.rb

@@ -6,6 +6,7 @@ module Floe
       class Task < Floe::Workflow::State
         include InputOutputMixin
         include NonTerminalMixin
+        include RetryCatchMixin
 
         attr_reader :credentials, :end, :heartbeat_seconds, :next, :parameters,
                     :result_selector, :resource, :timeout_seconds, :retry, :catch,
@@ -60,7 +61,8 @@ module Floe
         end
 
         def running?(context)
-          return true if waiting?(context)
+          return true  if waiting?(context)
+          return false if finished?(context)
 
           runner.status!(context.state["RunnerContext"])
           runner.running?(context.state["RunnerContext"])
@@ -82,54 +84,6 @@ module Floe
           runner.success?(context.state["RunnerContext"])
         end
 
-        def find_retrier(error)
-          self.retry.detect { |r| r.match_error?(error) }
-        end
-
-        def find_catcher(error)
-          self.catch.detect { |c| c.match_error?(error) }
-        end
-
-        def retry_state!(context, error)
-          retrier = find_retrier(error["Error"]) if error
-          return if retrier.nil?
-
-          # If a different retrier is hit reset the context
-          if !context["State"].key?("RetryCount") || context["State"]["Retrier"] != retrier.error_equals
-            context["State"]["RetryCount"] = 0
-            context["State"]["Retrier"]    = retrier.error_equals
-          end
-
-          context["State"]["RetryCount"] += 1
-
-          return if context["State"]["RetryCount"] > retrier.max_attempts
-
-          wait_until!(context, :seconds => retrier.sleep_duration(context["State"]["RetryCount"]))
-          context.next_state = context.state_name
-          context.output     = error
-          logger.info("Running state: [#{long_name}] with input [#{context.json_input}] got error[#{context.json_output}]...Retry - delay: #{wait_until(context)}")
-          true
-        end
-
-        def catch_error!(context, error)
-          catcher = find_catcher(error["Error"]) if error
-          return if catcher.nil?
-
-          context.next_state = catcher.next
-          context.output     = catcher.result_path.set(context.input, error)
-          logger.info("Running state: [#{long_name}] with input [#{context.json_input}]...CatchError - next state: [#{context.next_state}] output: [#{context.json_output}]")
-
-          true
-        end
-
-        def fail_workflow!(context, error)
-          # next_state is nil, and will be set to nil again in super
-          # keeping in here for completeness
-          context.next_state = nil
-          context.output = error
-          logger.error("Running state: [#{long_name}] with input [#{context.json_input}]...Complete workflow - output: [#{context.json_output}]")
-        end
-
         def parse_error(output)
           return if output.nil?
           return output if output.kind_of?(Hash)

data/lib/floe/workflow.rb

@@ -4,9 +4,8 @@ require "securerandom"
 require "json"
 
 module Floe
-  class Workflow
+  class Workflow < Floe::WorkflowBase
     include Logging
-    include ValidationMixin
 
     class << self
       def load(path_or_io, context = nil, credentials = {}, name = nil)
@@ -19,7 +18,6 @@ module Floe
 
       def wait(workflows, timeout: nil, &block)
         workflows = [workflows] if workflows.kind_of?(self)
-        logger.info("checking #{workflows.count} workflows...")
 
         run_until   = Time.now.utc + timeout if timeout.to_i > 0
         ready       = []
@@ -66,29 +64,20 @@ module Floe
             event, data = queue.pop
             break if event.nil?
 
-            _execution_id, runner_context = data.values_at("execution_id", "runner_context")
-
-            # If the event is for one of our workflows set the updated runner_context
-            workflows.each do |workflow|
-              next unless workflow.context.state.dig("RunnerContext", "container_ref") == runner_context["container_ref"]
-
-              workflow.context.state["RunnerContext"] = runner_context
-            end
-
-            break if queue.empty?
+            # break out of the loop if the event is for one of our workflows
+            break if queue.empty? || workflows.detect { |wf| wf.execution_id == data["execution_id"] }
           end
         ensure
           sleep_thread&.kill
         end
 
-        logger.info("checking #{workflows.count} workflows...Complete - #{ready.count} ready")
         ready
       ensure
         wait_thread&.kill
       end
     end
 
-    attr_reader :context, :payload, :states, :states_by_name, :start_at, :name, :comment
+    attr_reader :comment, :context
 
     def initialize(payload, context = nil, credentials = nil, name = nil)
       payload     = JSON.parse(payload)     if payload.kind_of?(String)
@@ -99,20 +88,10 @@ module Floe
       # caller should really put credentials into context and not pass that variable
       context.credentials = credentials if credentials
 
-      # NOTE: this is a string, and states use an array
-      @name        = name || "State Machine"
-      @payload     = payload
-      @context     = context
-      @comment     = payload["Comment"]
-      @start_at    = payload["StartAt"]
-
-      # NOTE: Everywhere else we include our name (i.e.: parent name) when building the child name.
-      #       When creating the states, we are dropping our name (i.e.: the workflow name)
-      @states      = payload["States"].to_a.map { |state_name, state| State.build!(self, ["States", state_name], state) }
+      @context = context
+      @comment = payload["Comment"]
 
-      validate_workflow
-
-      @states_by_name = @states.each_with_object({}) { |state, result| result[state.short_name] = state }
+      super(payload, name)
     rescue Floe::Error
       raise
     rescue => err
@@ -185,7 +164,7 @@ module Floe
 
     # NOTE: Expecting the context to be initialized (via start_workflow) before this
     def current_state
-      @states_by_name[context.state_name]
+      states_by_name[context.state_name]
     end
 
     # backwards compatibility. Caller should access directly from context
@@ -193,14 +172,12 @@ module Floe
       @context.credentials
     end
 
-    private
-
-    def validate_workflow
-      missing_field_error!("States") if @states.empty?
-      missing_field_error!("StartAt") if @start_at.nil?
-      invalid_field_error!("StartAt", @start_at, "is not found in \"States\"") unless workflow_state?(@start_at, self)
+    def execution_id
+      @context.execution["Id"]
     end
 
+    private
+
     def step!
       next_state = {"Name" => context.next_state, "Guid" => SecureRandom.uuid, "PreviousStateGuid" => context.state["Guid"]}
 

data/lib/floe/workflow_base.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Floe
+  class WorkflowBase
+    include ValidationMixin
+
+    attr_reader :name, :payload, :start_at, :states, :states_by_name
+
+    def initialize(payload, name = nil)
+      # NOTE: this is a string, and states use an array
+      @name     = name || "State Machine"
+      @payload  = payload
+      @start_at = payload["StartAt"]
+
+      # NOTE: Everywhere else we include our name (i.e.: parent name) when building the child name.
+      #       When creating the states, we are dropping our name (i.e.: the workflow name)
+      @states         = payload["States"].to_a.map { |state_name, state| Floe::Workflow::State.build!(self, ["States", state_name], state) }
+      @states_by_name = @states.to_h { |state| [state.short_name, state] }
+
+      validate_workflow!
+    end
+
+    def run(context)
+      run_nonblock(context) until context.ended?
+    end
+
+    def run_nonblock(context)
+      start_workflow(context)
+      loop while step_nonblock(context) == 0 && !context.ended?
+      self
+    end
+
+    def step_nonblock(context)
+      return Errno::EPERM if context.ended?
+
+      result = current_state(context).run_nonblock!(context)
+      return result if result != 0
+
+      context.state_history << context.state
+      context.next_state ? step!(context) : end_workflow!(context)
+
+      result
+    end
+
+    def step_nonblock_ready?(context)
+      !context.started? || current_state(context).ready?(context)
+    end
+
+    def waiting?(context)
+      current_state(context)&.waiting?(context)
+    end
+
+    def wait_until(context)
+      current_state(context)&.wait_until(context)
+    end
+
+    def start_workflow(context)
+      return if context.state_name
+
+      context.state["Name"]  = start_at
+      context.state["Input"] = context.execution["Input"].dup
+
+      context.execution["StartTime"] = Time.now.utc.iso8601
+
+      self
+    end
+
+    def current_state(context)
+      states_by_name[context.state_name]
+    end
+
+    def end?(context)
+      context.ended?
+    end
+
+    def output(context)
+      context.output.to_json if end?(context)
+    end
+
+    private
+
+    def step!(context)
+      next_state = {"Name" => context.next_state}
+
+      # if rerunning due to an error (and we are using Retry)
+      if context.state_name == context.next_state && context.failed? && context.state.key?("Retrier")
+        next_state.merge!(context.state.slice("RetryCount", "Input", "Retrier"))
+      else
+        next_state["Input"] = context.output
+      end
+
+      context.state = next_state
+    end
+
+    # Avoiding State#running? because that is potentially expensive.
+    # State#run_nonblock! already called running? via State#ready? and
+    # called State#finished -- which is what Context#state_finished? is detecting
+    def end_workflow!(context)
+      context.execution["EndTime"] = context.state["FinishedTime"]
+    end
+
+    def validate_workflow!
+      missing_field_error!("States") if @states.empty?
+      missing_field_error!("StartAt") if @start_at.nil?
+      invalid_field_error!("StartAt", @start_at, "is not found in \"States\"") unless workflow_state?(@start_at, self)
+    end
+  end
+end

data/lib/floe.rb

@@ -8,8 +8,11 @@ require_relative "floe/logging"
 require_relative "floe/runner"
 
 require_relative "floe/validation_mixin"
+require_relative "floe/workflow_base"
 require_relative "floe/workflow"
+# mixins used by workflow components
 require_relative "floe/workflow/error_matcher_mixin"
+require_relative "floe/workflow/branch"
 require_relative "floe/workflow/catcher"
 require_relative "floe/workflow/choice_rule"
 require_relative "floe/workflow/choice_rule/not"
@@ -17,6 +20,7 @@ require_relative "floe/workflow/choice_rule/or"
 require_relative "floe/workflow/choice_rule/and"
 require_relative "floe/workflow/choice_rule/data"
 require_relative "floe/workflow/context"
+require_relative "floe/workflow/item_processor"
 require_relative "floe/workflow/intrinsic_function"
 require_relative "floe/workflow/intrinsic_function/parser"
 require_relative "floe/workflow/intrinsic_function/transformer"
@@ -25,11 +29,14 @@ require_relative "floe/workflow/payload_template"
 require_relative "floe/workflow/reference_path"
 require_relative "floe/workflow/retrier"
 require_relative "floe/workflow/state"
+# mixins used by states
+require_relative "floe/workflow/states/child_workflow_mixin"
+require_relative "floe/workflow/states/input_output_mixin"
+require_relative "floe/workflow/states/non_terminal_mixin"
+require_relative "floe/workflow/states/retry_catch_mixin"
 require_relative "floe/workflow/states/choice"
 require_relative "floe/workflow/states/fail"
-require_relative "floe/workflow/states/input_output_mixin"
 require_relative "floe/workflow/states/map"
-require_relative "floe/workflow/states/non_terminal_mixin"
 require_relative "floe/workflow/states/parallel"
 require_relative "floe/workflow/states/pass"
 require_relative "floe/workflow/states/succeed"