durable_flow 0.1.0 → 0.2.0

data/lib/durable_flow/workflow.rb

@@ -10,6 +10,46 @@ module DurableFlow
 
     attr_reader :workflow_run
 
+    class << self
+      def retry_on(*exceptions, **options, &block)
+        return super(*exceptions, **options) unless block
+
+        super(*exceptions, **options) do |job, error|
+          begin
+            block.call(job, error)
+          ensure
+            job.send(:fail_workflow_after_unhandled_error!, error) if job.is_a?(DurableFlow::Workflow)
+          end
+        end
+      end
+
+      def discard_on(*exceptions, **options, &block)
+        super(*exceptions, **options) do |job, error|
+          begin
+            block.call(job, error) if block
+          ensure
+            job.send(:fail_workflow_after_unhandled_error!, error) if job.is_a?(DurableFlow::Workflow)
+          end
+        end
+      end
+    end
+
+    def perform_now
+      super
+    rescue Exception => error
+      fail_workflow_after_unhandled_error!(error)
+      raise
+    end
+
+    def retry_job(options = {})
+      if options[:error]
+        persist_retrying_workflow!(options[:error])
+        failed_workflow_step&.retry!(options[:error], retry_at: retry_at_from(options))
+      end
+
+      super
+    end
+
     def checkpoint!
       refresh_execution_lock!
       interrupt!(reason: :stopping) if queue_adapter.respond_to?(:stopping?) && queue_adapter.stopping?
@@ -25,7 +65,7 @@ module DurableFlow
       durable_step(name) do
         step_record = current_workflow_step
         metadata = step_record.metadata_hash
-        wake_at = parse_time(metadata["wake_at"]) || time_from(duration, explicit_time: until_time)
+        wake_at = durable_flow_parse_time(metadata["wake_at"]) || durable_flow_time_from(duration, explicit_time: until_time)
 
         raise ArgumentError, "Provide a duration or until: time for sleep step #{name.inspect}" unless wake_at
 
@@ -39,14 +79,15 @@ module DurableFlow
       end
     end
 
-    def wait_for_event_step(name, event_name:, timeout:, match:)
+    def wait_for_event_step(name, event_name:, timeout:, match:, allow_past_events: false, &block)
       durable_step(name) do
         step_record = current_workflow_step
         wait = find_or_initialize_wait(step_record, event_name: event_name, timeout: timeout, match: match)
 
-        if (event = matched_event_for(wait))
+        if (event = matched_event_for(wait, allow_past_events: allow_past_events))
           wait.update!(status: "matched", workflow_event: event)
-          event.payload_value
+          payload = event.payload_value
+          block ? block.call(payload) : payload
         elsif wait.timeout_at && Time.current >= wait.timeout_at
           wait.update!(status: "timed_out")
           step_record.update!(status: "failed", metadata: step_record.metadata_hash.merge("timeout_at" => wait.timeout_at.utc.iso8601(9)))
@@ -73,12 +114,83 @@ module DurableFlow
       StepProxy.new(self).wait_for_workflow(name, workflow_or_run_id, timeout: timeout)
     end
 
+    def child_workflow(name, workflow_class = nil, *args, timeout: nil, on_failure: :raise, **kwargs, &block)
+      validate_child_workflow_failure_policy!(on_failure)
+
+      base_name = name.to_s
+      child = start_child_workflow_step("#{base_name}_start", workflow_class, *args, **kwargs, &block)
+
+      wait_for_child_workflow("#{base_name}_wait", child, timeout: timeout, on_failure: on_failure)
+    end
+
+    def invoke_workflow(name, workflow_class = nil, *args, timeout: nil, on_failure: :raise, **kwargs, &block)
+      child_workflow(name, workflow_class, *args, timeout: timeout, on_failure: on_failure, **kwargs, &block)
+    end
+
+    def invoke_workflows(name, collection, timeout: nil, concurrency: nil, on_failure: :raise, &block)
+      validate_child_workflow_failure_policy!(on_failure)
+      raise ArgumentError, "Provide a block that returns workflow requests" unless block
+
+      requests = collection.map do |item|
+        request = block.call(item)
+        unless request.respond_to?(:workflow_key)
+          raise ArgumentError, "invoke_each blocks must return a workflow request with a stable workflow_key"
+        end
+
+        request
+      end
+
+      child_workflows(name, requests, timeout: timeout, concurrency: concurrency, on_failure: on_failure)
+    end
+
+    def child_workflows(name, collection = nil, key: nil, timeout: nil, concurrency: nil, on_failure: :raise, &block)
+      validate_child_workflow_failure_policy!(on_failure)
+
+      if collection.nil?
+        raise ArgumentError, "Provide a child workflow collection or builder block" unless block
+
+        builder = ChildWorkflowBuilder.new
+        block.call(builder)
+        collection = builder.requests
+        block = nil
+        key ||= :workflow_key
+      end
+
+      batches = child_workflow_batches(collection, concurrency)
+
+      batches.flat_map do |batch|
+        children = batch.map do |item|
+          item_key = child_workflow_item_key(item, key)
+          child = start_child_workflow_step("#{name}_#{item_key}_start") do
+            block ? block.call(item) : start_child_workflow_request(item)
+          end
+
+          child.merge("key" => item_key)
+        end
+
+        children.map do |child|
+          completion = wait_for_child_workflow("#{name}_#{child.fetch("key")}_wait", child, timeout: timeout, on_failure: on_failure)
+          completion.to_h.with_indifferent_access.merge(
+            "key" => child.fetch("key"),
+            "run_id" => child.fetch("run_id"),
+            "workflow_class" => child["workflow_class"] || completion_value(completion, :workflow_class),
+          ).compact
+        end
+      end
+    end
+
+    def each_child_workflow(name, collection, key:, timeout: nil, on_failure: :raise, &block)
+      raise ArgumentError, "Provide a block that starts each child workflow" unless block
+
+      child_workflows(name, collection, key: key, timeout: timeout, on_failure: on_failure, &block)
+    end
+
     def log
       @workflow_logger ||= WorkflowLogger.new(self)
     end
 
     private
-      attr_reader :current_workflow_step
+      attr_reader :current_workflow_step, :failed_workflow_step
 
       def continue(&block)
         ensure_workflow_run!
@@ -94,20 +206,18 @@ module DurableFlow
           end
 
           mark_workflow_running!
-          block.call
-          complete_workflow!
+          result = block.call
+          complete_workflow!(result)
         rescue Pause => pause
           persist_interrupted_workflow!(pause.status)
         rescue ActiveJob::Continuation::Interrupt => interrupt
           resume_job(interrupt)
         rescue ActiveJob::Continuation::Error => error
-          fail_workflow!(error)
           raise
         rescue StandardError => error
           if resume_errors_after_advancing? && continuation.advanced?
             resume_job(exception: error)
           else
-            fail_workflow!(error)
             raise
           end
         ensure
@@ -150,6 +260,12 @@ module DurableFlow
               value = block.arity == 0 ? block.call : block.call(continuation_step)
               step_record.complete!(value)
               loaded = true
+            rescue Pause, ActiveJob::Continuation::Interrupt
+              raise
+            rescue StandardError => error
+              @failed_workflow_step = step_record
+              step_record.fail!(error)
+              raise
             ensure
               @current_workflow_step = nil
             end
@@ -219,20 +335,23 @@ module DurableFlow
             event_name: event_name.to_s,
             status: "pending",
             match: Serializer.dump(match || {}),
-            timeout_at: (time_from(timeout) if timeout),
+            timeout_at: (durable_flow_time_from(timeout) if timeout),
           )).tap do |wait|
           updates = {}
           updates[:event_name] = event_name.to_s if wait.event_name != event_name.to_s
           updates[:match] = Serializer.dump(match || {}) if wait.match.blank?
-          updates[:timeout_at] = time_from(timeout) if timeout && wait.timeout_at.blank?
+          updates[:timeout_at] = durable_flow_time_from(timeout) if timeout && wait.timeout_at.blank?
           wait.update!(updates) if updates.any?
         end
       end
 
-      def matched_event_for(wait)
+      def matched_event_for(wait, allow_past_events: false)
         return wait.workflow_event if wait.workflow_event
 
-        WorkflowEvent.named(wait.event_name).where("created_at >= ?", wait.created_at).order(:created_at).detect do |event|
+        scope = WorkflowEvent.named(wait.event_name)
+        scope = scope.where("created_at >= ?", wait.created_at) unless allow_past_events
+
+        scope.order(:created_at).detect do |event|
           wait.matches_event?(event)
         end
       end
@@ -307,7 +426,20 @@ module DurableFlow
         )
       end
 
-      def complete_workflow!
+      def persist_retrying_workflow!(error)
+        ensure_workflow_run!
+
+        workflow_run.update!(
+          status: "retrying",
+          interrupted_at: Time.current,
+          serialized_job: serialize,
+          queue_name: queue_name,
+          priority: priority,
+          last_error: error_payload(error),
+        )
+      end
+
+      def complete_workflow!(result = nil)
         workflow_run.update!(
           status: "completed",
           completed_at: Time.current,
@@ -315,11 +447,23 @@ module DurableFlow
           last_error: nil,
         )
 
-        DurableFlow.notify(DurableFlow::WORKFLOW_COMPLETED_EVENT, {
+        payload = {
           run_id: workflow_run.run_id,
           job_id: job_id,
           workflow_class: self.class.name,
-        })
+          result: result,
+        }
+
+        DurableFlow.notify(DurableFlow::WORKFLOW_COMPLETED_EVENT, payload)
+        DurableFlow.notify(DurableFlow::WORKFLOW_FINISHED_EVENT, payload.merge(status: "completed"))
+      end
+
+      def fail_workflow_after_unhandled_error!(error)
+        return unless workflow_run
+        return if workflow_run.terminal?
+
+        failed_workflow_step&.fail!(error)
+        fail_workflow!(error)
       end
 
       def fail_workflow!(error)
@@ -327,23 +471,182 @@ module DurableFlow
           status: "failed",
           failed_at: Time.current,
           serialized_job: serialize,
-          last_error: {
-            "class" => error.class.name,
-            "message" => error.message,
-            "backtrace" => Array(error.backtrace).first(10),
-          },
+          last_error: error_payload(error),
         )
 
-        DurableFlow.notify(DurableFlow::WORKFLOW_FAILED_EVENT, {
+        payload = {
           run_id: workflow_run.run_id,
           job_id: job_id,
           workflow_class: self.class.name,
           error_class: error.class.name,
           error_message: error.message,
-        })
+        }
+
+        DurableFlow.notify(DurableFlow::WORKFLOW_FAILED_EVENT, payload)
+        DurableFlow.notify(DurableFlow::WORKFLOW_FINISHED_EVENT, payload.merge(status: "failed"))
+      end
+
+      def error_payload(error)
+        {
+          "class" => error.class.name,
+          "message" => error.message,
+          "backtrace" => Array(error.backtrace).first(10),
+        }
+      end
+
+      def retry_at_from(options)
+        return options[:wait_until].to_time if options[:wait_until].respond_to?(:to_time)
+        return unless options[:wait]
+
+        Time.current + options[:wait].to_i
+      end
+
+      def start_child_workflow_step(name, workflow_class = nil, *args, **kwargs, &block)
+        durable_step(name) do
+          child = if block
+            block.arity.zero? ? block.call : block.call(workflow_class)
+          else
+            raise ArgumentError, "Provide a workflow class or block for child workflow #{name.inspect}" unless workflow_class
+
+            workflow_class.perform_later(*args, **kwargs)
+          end
+
+          child_workflow_start_payload(child, workflow_class)
+        end
+      end
+
+      def wait_for_child_workflow(name, child, timeout: nil, on_failure: :raise)
+        validate_child_workflow_failure_policy!(on_failure)
+
+        child = child.to_h.stringify_keys
+        wait_for_event_step(
+          name,
+          event_name: DurableFlow::WORKFLOW_FINISHED_EVENT,
+          timeout: timeout,
+          match: { run_id: child.fetch("run_id") },
+          allow_past_events: true,
+        ) do |completion|
+          if completion_value(completion, :status) == "failed" && on_failure == :raise
+            raise_child_workflow_failed!(completion)
+          end
+
+          completion
+        end
+      end
+
+      def validate_child_workflow_failure_policy!(value)
+        return if %i[raise return].include?(value)
+
+        raise ArgumentError, "Child workflow on_failure must be :raise or :return"
+      end
+
+      def child_workflow_start_payload(child, workflow_class = nil)
+        if child.respond_to?(:to_h)
+          payload = child.to_h.with_indifferent_access
+          if payload[:run_id].present?
+            return {
+              "run_id" => payload.fetch(:run_id),
+              "workflow_class" => payload[:workflow_class],
+            }.compact
+          end
+        end
+
+        {
+          "run_id" => extract_child_workflow_run_id(child),
+          "workflow_class" => child_workflow_class_name(child, workflow_class),
+        }.compact
+      end
+
+      def extract_child_workflow_run_id(child)
+        run_id = child.respond_to?(:job_id) ? child.job_id : child.to_s
+
+        raise ArgumentError, "Child workflow start must return a job, run id, or object responding to job_id" if run_id.blank?
+
+        run_id
+      end
+
+      def child_workflow_item_key(item, key)
+        key ||= :workflow_key
+
+        value = if key.respond_to?(:call)
+          key.call(item)
+        elsif item.respond_to?(:fetch)
+          item.fetch(key) { item.fetch(key.to_s) }
+        else
+          item.public_send(key)
+        end
+
+        value.to_s
+      end
+
+      def child_workflow_batches(collection, concurrency)
+        items = collection.to_a
+        return [ items ] unless concurrency
+
+        size = concurrency.to_i
+        raise ArgumentError, "Child workflow concurrency must be at least 1" if size < 1
+
+        items.each_slice(size).to_a
+      end
+
+      def start_child_workflow_request(request)
+        if request.respond_to?(:workflow_class)
+          workflow_class = request.workflow_class
+          args = request.respond_to?(:workflow_args) ? Array(request.workflow_args) : []
+          kwargs = child_workflow_request_kwargs(request)
+
+          return child_workflow_start_payload(workflow_class.perform_later(*args, **kwargs), workflow_class)
+        end
+
+        unless request.respond_to?(:perform_later)
+          raise ArgumentError, "Child workflow request must respond to perform_later or workflow_class"
+        end
+
+        request.perform_later
+      end
+
+      def child_workflow_request_kwargs(request)
+        kwargs = if request.respond_to?(:workflow_kwargs)
+          request.workflow_kwargs
+        elsif request.respond_to?(:workflow_arguments)
+          request.workflow_arguments
+        else
+          {}
+        end
+
+        kwargs ||= {}
+        kwargs.respond_to?(:symbolize_keys) ? kwargs.symbolize_keys : kwargs.to_h
+      end
+
+      def child_workflow_class_name(child, workflow_class = nil)
+        return workflow_class.name if workflow_class.respond_to?(:name)
+        return child.class.name if child.respond_to?(:job_id)
+
+        nil
+      end
+
+      def raise_child_workflow_failed!(completion)
+        raise ChildWorkflowFailedError.new(
+          run_id: completion_value(completion, :run_id),
+          workflow_class: completion_value(completion, :workflow_class),
+          error_class: completion_value(completion, :error_class),
+          error_message: completion_value(completion, :error_message),
+        )
+      end
+
+      def completion_value(payload, key)
+        return unless payload.respond_to?(:[])
+
+        if payload.respond_to?(:key?) && payload.key?(key)
+          payload[key]
+        elsif payload.respond_to?(:key?) && payload.key?(key.to_s)
+          payload[key.to_s]
+        else
+          payload[key]
+        end
       end
 
-      def time_from(value, explicit_time: nil)
+      def durable_flow_time_from(value, explicit_time: nil)
         return explicit_time.to_time if explicit_time.respond_to?(:to_time)
         return nil if value.nil?
         return value.to_time if value.respond_to?(:to_time)
@@ -351,7 +654,7 @@ module DurableFlow
         Time.current + value
       end
 
-      def parse_time(value)
+      def durable_flow_parse_time(value)
         return if value.blank?
         return value if value.is_a?(Time)
 

data/lib/durable_flow.rb

@@ -6,6 +6,8 @@ require "active_job/continuation"
 require "active_record"
 require "active_support/core_ext/numeric/time"
 require "active_support/core_ext/object/blank"
+require "active_support/core_ext/hash/indifferent_access"
+require "active_support/core_ext/hash/keys"
 require "active_support/core_ext/module/attribute_accessors"
 require "securerandom"
 
@@ -14,8 +16,11 @@ require "durable_flow/errors"
 require "durable_flow/serializer"
 require "durable_flow/schema"
 require "durable_flow/live"
+require "durable_flow/definition_graph"
+require "durable_flow/definition_analyzer"
 require "durable_flow/workflow_logger"
 require "durable_flow/workflow_timeline"
+require "durable_flow/child_workflow_builder"
 require "durable_flow/models/application_record"
 require "durable_flow/models/workflow_run"
 require "durable_flow/models/workflow_step"
@@ -39,6 +44,7 @@ module DurableFlow
 
   WORKFLOW_COMPLETED_EVENT = "durable_flow.workflow.completed"
   WORKFLOW_FAILED_EVENT = "durable_flow.workflow.failed"
+  WORKFLOW_FINISHED_EVENT = "durable_flow.workflow.finished"
   IGNORED_EVENT_NAMESPACES = %w[
     action_controller
     action_mailbox

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: durable_flow
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - DurableFlow contributors
@@ -89,6 +89,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '9.0'
+- !ruby/object:Gem::Dependency
+  name: prism
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
 - !ruby/object:Gem::Dependency
   name: railties
   requirement: !ruby/object:Gem::Requirement
@@ -162,11 +176,15 @@ files:
 - MIT-LICENSE
 - README.md
 - app/controllers/durable_flow/workflow_runs_controller.rb
+- app/views/durable_flow/workflow_runs/definition.html.erb
 - app/views/durable_flow/workflow_runs/index.html.erb
 - app/views/durable_flow/workflow_runs/show.html.erb
 - app/views/layouts/durable_flow/application.html.erb
 - config/routes.rb
 - lib/durable_flow.rb
+- lib/durable_flow/child_workflow_builder.rb
+- lib/durable_flow/definition_analyzer.rb
+- lib/durable_flow/definition_graph.rb
 - lib/durable_flow/dispatcher.rb
 - lib/durable_flow/engine.rb
 - lib/durable_flow/errors.rb