geneva_drive 0.6.0 → 0.7.0

data/lib/geneva_drive/test_helpers.rb

@@ -46,6 +46,8 @@ module GenevaDrive::TestHelpers
       step_execution = workflow.current_execution
       break unless step_execution
 
+      raise_if_parked!(step_execution, "speedrun_workflow")
+
       # interruptible: false so resumable steps run to completion
       step_execution.execute!(interruptible: false)
       iterations += 1
@@ -79,6 +81,8 @@ module GenevaDrive::TestHelpers
     step_execution = workflow.current_execution
     return nil unless step_execution
 
+    raise_if_parked!(step_execution, "perform_next_step")
+
     step_execution.execute!
     workflow.reload
     step_execution
@@ -117,8 +121,9 @@ module GenevaDrive::TestHelpers
         "Available steps: #{available}"
     end
 
-    # Cancel any existing scheduled step executions to satisfy uniqueness constraint
-    workflow.step_executions.where(state: "scheduled").update_all(
+    # Cancel any existing scheduled or parked step executions to satisfy the
+    # uniqueness constraint
+    workflow.step_executions.where(state: %w[scheduled waiting]).update_all(
       state: "canceled",
       outcome: "canceled",
       canceled_at: Time.current
@@ -194,11 +199,14 @@ module GenevaDrive::TestHelpers
     step_execution = workflow.current_execution
     return nil unless step_execution
 
+    raise_if_parked!(step_execution, "speedrun_current_step")
+
     executions = 0
     original_step_name = step_execution.step_name
 
     loop do
       step_execution.reload
+      raise_if_parked!(step_execution, "speedrun_current_step")
 
       # Follow the execution chain: a completed execution that spawned a
       # successor means the step is still going (suspend! and skip_to! create
@@ -289,4 +297,83 @@ module GenevaDrive::TestHelpers
     assert successor, "Expected successor execution for step #{step_name}, but none found"
     assert successor.scheduled?, "Expected successor to be scheduled, but was #{successor.state}"
   end
+
+  # === Signal Test Helpers ===
+
+  # Asserts that the workflow is parked waiting for a signal.
+  #
+  # @param workflow [GenevaDrive::Workflow] the workflow to check
+  # @param signal_name [String, Symbol, nil] the signal name the step waits for
+  # @return [void]
+  #
+  # @example Check that the workflow is waiting
+  #   assert_waiting_for_signal(workflow, :payment_confirmed)
+  #
+  def assert_waiting_for_signal(workflow, signal_name = nil)
+    workflow.reload
+    execution = workflow.step_executions.where(state: "waiting").first
+
+    assert execution,
+      "Expected #{workflow.class.name} to be waiting for a signal, but no parked step execution was found " \
+      "(current execution: #{workflow.current_execution&.state.inspect})"
+
+    return unless signal_name
+
+    matcher = execution.step_definition&.signal_matcher
+    assert_equal signal_name.to_s, matcher_name(matcher),
+      "Expected step '#{execution.step_name}' to be waiting for #{signal_name.inspect}, " \
+      "but it waits for #{matcher}"
+  end
+
+  # Asserts that a signal with the given name exists in the given state.
+  #
+  # @param workflow [GenevaDrive::Workflow] the workflow to check
+  # @param signal_name [String, Symbol] the signal name
+  # @param state [String, Symbol] the expected signal state
+  # @return [void]
+  #
+  # @example Check that the signal was consumed
+  #   assert_signal_state(workflow, :payment_confirmed, :consumed)
+  #
+  def assert_signal_state(workflow, signal_name, state)
+    signal = workflow.signals.where(name: signal_name.to_s).order(:created_at, :id).last
+
+    assert signal, "Expected a #{signal_name.inspect} signal on #{workflow.class.name}, but none was found"
+    assert_equal state.to_s, signal.state,
+      "Expected signal #{signal_name.inspect} to be #{state}, but was #{signal.state}"
+  end
+
+  private
+
+  # The signal name a matcher accepts, when it has one to give. Custom
+  # matchers own their whole predicate and need not expose a name.
+  #
+  # @param matcher [Object, nil] anything responding to #matches?(signal)
+  # @return [String, nil]
+  def matcher_name(matcher)
+    matcher.name if matcher.respond_to?(:name)
+  end
+
+  # Raises a descriptive error when a step-driving helper runs into a parked
+  # execution. Without this a parked workflow would look like an infinite
+  # loop (or a silently passing assertion) instead of "you forgot to signal".
+  #
+  # @param step_execution [GenevaDrive::StepExecution] the execution about to be driven
+  # @param helper_name [String] the helper that was called
+  # @return [void]
+  def raise_if_parked!(step_execution, helper_name)
+    return unless step_execution.waiting?
+
+    matcher = begin
+      step_execution.step_definition&.signal_matcher
+    rescue
+      nil
+    end
+    waiting_for = matcher ? " waiting for #{matcher}" : ""
+    example_name = matcher_name(matcher)&.to_sym
+
+    raise "#{helper_name} cannot drive step '#{step_execution.step_name}': its execution is parked" \
+          "#{waiting_for}. Deliver the signal first, e.g. " \
+          "workflow.signal!(#{example_name.inspect}), then call #{helper_name} again."
+  end
 end

data/lib/geneva_drive/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module GenevaDrive
-  VERSION = "0.6.0"
+  VERSION = "0.7.0"
 end

data/lib/geneva_drive/workflow.rb

@@ -43,6 +43,11 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     foreign_key: :workflow_id,
     inverse_of: :workflow,
     dependent: :delete_all
+  has_many :signals,
+    class_name: "GenevaDrive::Signal",
+    foreign_key: :workflow_id,
+    inverse_of: :workflow,
+    dependent: :delete_all
 
   # Class-inheritable attributes for DSL
   class_attribute :_step_definitions, instance_writer: false, default: []
@@ -523,7 +528,14 @@ class GenevaDrive::Workflow < ActiveRecord::Base
 
     # A scheduled execution preserved by pause! takes precedence - re-enqueue it
     scheduled_execution = current_execution
-    return enqueue_scheduled_execution(scheduled_execution) if scheduled_execution
+    if scheduled_execution
+      # A parked execution is not re-enqueued: it is waiting for an event, not
+      # for a clock. Re-run the rendezvous instead, which delivers any signal
+      # that arrived while the workflow was paused.
+      return rendezvous_waiting_execution!(scheduled_execution) if scheduled_execution.waiting?
+
+      return enqueue_scheduled_execution(scheduled_execution)
+    end
 
     # Resumable-step continuations need the cursor/continues_from_id columns
     if GenevaDrive::StepExecution.resumable_columns?
@@ -557,12 +569,206 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     create_step_execution(step_def, wait: nil)
   end
 
+  # Delivers an external event to this workflow.
+  #
+  # The signal row is persisted first and dispatched second, which is what
+  # makes arrival order irrelevant: a signal that lands before the waiting
+  # step's execution even exists simply sits in the table until that step's
+  # gate picks it up, and a signal that lands while a step is already parked
+  # wakes it immediately.
+  #
+  # Pass an +idempotency_key+ (a webhook's event id, typically) to make
+  # redelivery a no-op: the second call returns the row the first one wrote,
+  # flagged {GenevaDrive::Signal#duplicate_delivery?}, without dispatching.
+  #
+  # On a paused workflow the row is persisted but not dispatched - waking a
+  # step while the workflow is paused would only get its execution canceled.
+  # {#resume!} performs the rendezvous instead.
+  #
+  # @param signal_name [Symbol, String] the event name
+  # @param payload [Object] data describing the event, serialized through ActiveJob
+  # @param idempotency_key [String, nil] deduplication key, scoped to (workflow, name)
+  # @return [GenevaDrive::Signal] the persisted signal
+  # @raise [ArgumentError] if the signal name is blank
+  # @raise [WorkflowNotOngoing] if the workflow is finished or canceled and this is a new event
+  # @raise [SignalPayloadTooLargeError] if the payload exceeds GenevaDrive.max_signal_payload_size
+  #
+  # @example Deliver a webhook event
+  #   workflow = OrderFulfillmentWorkflow.ongoing.for_hero(order).first
+  #   workflow.signal!(:payment_confirmed,
+  #     payload: {amount_cents: 12_500},
+  #     idempotency_key: event["id"])
+  def signal!(signal_name, payload: {}, idempotency_key: nil, **unknown_options)
+    if unknown_options.any?
+      raise ArgumentError,
+        "Unknown options passed to signal!: #{unknown_options.keys.join(", ")}"
+    end
+
+    if signal_name.blank?
+      raise ArgumentError, "signal! requires a signal name"
+    end
+
+    name = signal_name.to_s
+    dedup_key = idempotency_key&.to_s
+
+    # Serialize (and bound) before anything is persisted.
+    serialized_payload = GenevaDrive::Signal.serialize_payload(payload)
+
+    signal = nil
+    duplicate = false
+
+    # One transaction covers the lot: the INSERT, the waiting-execution scan,
+    # and every attach, flip and counter bump. Taking the lock before writing
+    # anything is what makes delivery all-or-nothing - insert first and
+    # dispatch second, and a crash in between would leave a pending signal
+    # sitting next to a waiting execution with nothing left to introduce them.
+    with_lock do
+      # with_lock reloads; the terminal check belongs inside it
+      unless ongoing?
+        existing = dedup_key && signals.find_by(name: name, idempotency_key: dedup_key)
+        unless existing
+          raise GenevaDrive::WorkflowNotOngoing,
+            "Cannot deliver signal #{name.inspect} to a #{state} workflow"
+        end
+
+        logger.info("Signal #{name.inspect} redelivered to #{state} workflow, returning existing row")
+        signal = existing
+        duplicate = true
+        next
+      end
+
+      begin
+        # A savepoint so that hitting the dedup index does not poison the
+        # transaction - the caller's, if they had one open, or ours.
+        transaction(requires_new: true) do
+          record = signals.new(name: name, idempotency_key: dedup_key, state: "pending")
+          record[:payload] = serialized_payload
+          record.save!
+          signal = record
+        end
+      rescue ActiveRecord::RecordNotUnique
+        signal = signals.find_by!(name: name, idempotency_key: dedup_key)
+        logger.info("Signal #{name.inspect} is a duplicate delivery of signal #{signal.id}, not dispatching")
+        duplicate = true
+        next
+      end
+
+      logger.info("Received signal #{name.inspect} as signal #{signal.id}")
+
+      if paused?
+        logger.info("Workflow is paused, buffering signal #{signal.id} until resume!")
+      else
+        wake_executions_matching!(signal)
+      end
+    end
+
+    signal.duplicate_delivery! if duplicate
+    signal
+  end
+
   # Returns the current active step execution, if any.
-  # Includes scheduled and in_progress states.
+  # Includes scheduled, waiting and in_progress states.
   #
   # @return [StepExecution, nil] the current execution
   def current_execution
-    step_executions.where(state: %w[scheduled in_progress]).first
+    step_executions.where(state: %w[scheduled waiting in_progress]).first
+  end
+
+  # Signals that may still be attached to an execution, oldest first, each
+  # pointing back at this workflow instance.
+  #
+  # Matcher blocks are instance_exec'd on `signal.workflow`, and `inverse_of:`
+  # only primes records loaded straight off the association - a scoped
+  # relation would hand each signal a freshly loaded workflow, which would
+  # then see none of the caller's unsaved state.
+  #
+  # @return [Array<GenevaDrive::Signal>]
+  # @api private
+  public def attachable_signals
+    signals.attachable.map { |signal| adopt_signal(signal) }
+  end
+
+  # Wakes every parked step execution whose matcher accepts the given signal.
+  #
+  # Runs under the workflow row lock - the same lock create_step_execution
+  # and the Executor take - so whichever of dispatch and the gate commits
+  # second sees the other's write and no signal can slip through the gap.
+  #
+  # @param signal [GenevaDrive::Signal] the signal to deliver
+  # @return [Array<StepExecution>] the executions that were woken
+  # @api private
+  public def dispatch_signal!(signal)
+    return [] unless GenevaDrive::StepExecution.signal_columns?
+
+    with_lock { wake_executions_matching!(signal) }
+  end
+
+  # Settles the signal attached to an execution we are deliberately moving
+  # past, wherever the skip came from: `skip_if` firing at wake, flow control
+  # inside the step, an exception policy, or an operator calling `skip!`.
+  #
+  # Consumption follows attachment. A skip in any flavor says "we are done
+  # with this step", so an event that was delivered to it is spent by that
+  # decision - leaving it claimed would hand a stale event to the next waiter
+  # for the same name, which is exactly what consume-once-per-event forbids.
+  # An execution that never attached has nothing to settle, so buffered
+  # signals a step never reached are left alone.
+  #
+  # Must be called with the workflow lock held, so the settle rides the same
+  # transaction as the skip itself.
+  #
+  # @param step_execution [StepExecution, nil] the execution being skipped past
+  # @return [GenevaDrive::Signal, nil] the settled signal, if there was one
+  # @api private
+  public def settle_signal_for_skipped!(step_execution)
+    return unless GenevaDrive::StepExecution.signal_columns?
+    return if step_execution.nil? || step_execution.signal_id.blank?
+
+    signal = step_execution.signal
+    return unless signal && signal.state == "claimed"
+
+    logger.info("Skipping past step #{step_execution.step_name}, settling attached signal #{signal.id} (#{signal.name})")
+    signal.record_consumption!(resolved_step_names: [step_execution.step_name])
+    signal
+  end
+
+  # The body of dispatch, for callers that already hold the workflow lock
+  # (`signal!` holds it across the whole delivery so the rows land atomically).
+  #
+  # @param signal [GenevaDrive::Signal] the signal to deliver
+  # @return [Array<StepExecution>] the executions that were woken
+  # @api private
+  public def wake_executions_matching!(signal)
+    return [] unless GenevaDrive::StepExecution.signal_columns?
+    return [] if paused? || !ongoing?
+
+    # Matcher blocks run on signal.workflow, so point the signal at this very
+    # instance instead of letting it load a second copy from the database.
+    adopt_signal(signal)
+    woken = []
+
+    step_executions.where(state: "waiting").order(created_at: :asc, id: :asc).each do |execution|
+      step_def = execution.step_definition
+      # A step whose definition is gone (class removed, step renamed) can
+      # never match - the matcher lives on the definition.
+      next unless step_def&.waits_for_signal?
+      next unless step_def.signal_matcher.matches?(signal)
+
+      signal.claim!
+      execution.update!(
+        signal_id: signal.id,
+        state: "scheduled",
+        scheduled_for: Time.current,
+        waiting_since: nil
+      )
+      woken << [execution, step_def]
+    end
+
+    woken.each do |execution, step_def|
+      enqueue_woken_execution(execution, step_def)
+    end
+
+    woken.map(&:first)
   end
 
   # Returns all step executions in chronological order.
@@ -732,6 +938,64 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     create_step_execution(first_step, wait: first_step.wait)
   end
 
+  # Points a signal's workflow association at this instance, so matcher
+  # blocks - which are instance_exec'd on it - see the live object.
+  #
+  # @param signal [GenevaDrive::Signal]
+  # @return [GenevaDrive::Signal] the same signal
+  def adopt_signal(signal)
+    signal.association(:workflow).target = self
+    signal
+  end
+
+  # Enqueues the PerformStepJob for an execution that dispatch just flipped
+  # from waiting to scheduled. Uses the same enqueue discipline (merged job
+  # options, after-commit deferral, job_id writeback) as create_step_execution.
+  #
+  # @param step_execution [StepExecution] the woken execution
+  # @param step_definition [StepDefinition] its step definition
+  # @return [void]
+  def enqueue_woken_execution(step_execution, step_definition)
+    job_options = merged_step_job_options(step_definition)
+    execution_id = step_execution.id
+    workflow_logger = logger
+
+    run_after_commit do
+      job = GenevaDrive::PerformStepJob
+        .set(**job_options)
+        .perform_later(execution_id)
+
+      workflow_logger.debug("Enqueued PerformStepJob with job_id=#{job.job_id} for woken step execution #{execution_id}")
+
+      GenevaDrive::StepExecution
+        .where(id: execution_id)
+        .update_all(job_id: job.job_id)
+    end
+  end
+
+  # Re-runs the rendezvous for a step execution that is parked waiting for a
+  # signal. Signals that arrived while the workflow was paused are delivered
+  # here, on resume.
+  #
+  # @param step_execution [StepExecution] the parked execution
+  # @return [StepExecution] the same execution (rescheduled if a signal matched)
+  def rendezvous_waiting_execution!(step_execution)
+    step_def = step_execution.step_definition
+    matcher = step_def&.signal_matcher
+
+    if matcher
+      candidate = attachable_signals.detect { |signal| matcher.matches?(signal) }
+      if candidate
+        logger.info("Resuming into a matching signal #{candidate.id} (#{candidate.name}) for step #{step_execution.step_name}")
+        dispatch_signal!(candidate)
+        return step_execution.reload
+      end
+    end
+
+    logger.info("Step #{step_execution.step_name} stays parked - no matching signal has arrived yet")
+    step_execution
+  end
+
   # Enqueues a job for an existing scheduled execution.
   #
   # If the execution is overdue (scheduled_for is in the past), runs immediately.
@@ -785,22 +1049,24 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     scheduled_for = wait ? wait.from_now : Time.current
 
     with_lock do
-      # Cancel any stray scheduled executions - the successor is the one
-      # execution that should run next (same as create_step_execution).
-      canceled_count = step_executions.scheduled.update_all(
-        state: "canceled",
-        outcome: "canceled",
-        canceled_at: Time.current
-      )
-      logger.debug("Canceled #{canceled_count} previously scheduled step execution(s)") if canceled_count > 0
+      # Cancel any stray scheduled or parked executions - the successor is the
+      # one execution that should run next (same as create_step_execution).
+      cancel_stray_executions!
 
-      successor = step_executions.create!(
+      successor_attributes = {
         step_name: predecessor.step_name,
         state: "scheduled",
         scheduled_for: scheduled_for,
         continues_from_id: predecessor.id,
         cursor: predecessor.cursor
-      )
+      }
+      # Carry the attachment pin across the chain so successors never re-park
+      # and received_signal stays stable for the whole iteration.
+      if GenevaDrive::StepExecution.signal_columns?
+        successor_attributes[:signal_id] = predecessor.signal_id
+      end
+
+      successor = step_executions.create!(**successor_attributes)
 
       # next_step_name points to the step that's scheduled to run next
       update!(next_step_name: predecessor.step_name)
@@ -840,15 +1106,7 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     scheduled_for = wait ? wait.from_now : Time.current
 
     with_lock do
-      # Cancel any scheduled step executions (not in_progress - those are being executed).
-      # Safe to use update_all since we hold the workflow lock, blocking any executor
-      # that would try to start these steps.
-      canceled_count = step_executions.scheduled.update_all(
-        state: "canceled",
-        outcome: "canceled",
-        canceled_at: Time.current
-      )
-      logger.debug("Canceled #{canceled_count} previously scheduled step execution(s)") if canceled_count > 0
+      cancel_stray_executions!
 
       step_execution = step_executions.create!(
         step_name: step_definition.name,
@@ -892,6 +1150,26 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     end
   end
 
+  # Cancels any scheduled or parked step executions so that the execution
+  # about to be created is the only one that will run. In-progress executions
+  # are left alone - they are being executed.
+  #
+  # Safe to use update_all because the caller holds the workflow lock, which
+  # blocks any executor that would try to start these steps. Parked executions
+  # are swept for the same reason: no legitimate path creates a new execution
+  # past one that is waiting.
+  #
+  # @return [Integer] the number of executions canceled
+  def cancel_stray_executions!
+    canceled_count = step_executions.where(state: %w[scheduled waiting]).update_all(
+      state: "canceled",
+      outcome: "canceled",
+      canceled_at: Time.current
+    )
+    logger.debug("Canceled #{canceled_count} previously scheduled step execution(s)") if canceled_count > 0
+    canceled_count
+  end
+
   # Finishes the workflow.
   #
   # @return [nil]
@@ -948,6 +1226,33 @@ class GenevaDrive::Workflow < ActiveRecord::Base
     @tagged_logger = previous_tagged_logger
   end
 
+  # The signal that woke the step currently executing, or nil when the step
+  # does not declare +wait_for:+. Available inside step bodies only.
+  #
+  # @return [GenevaDrive::Signal, nil]
+  #
+  # @example Read the payload of the event that woke the step
+  #   step :capture, wait_for: :payment_confirmed do
+  #     hero.capture!(received_signal.payload[:amount_cents])
+  #   end
+  public attr_reader :received_signal
+
+  # Temporarily makes a signal available to step code as +received_signal+
+  # for the duration of the block. Injected by the Executor the same way the
+  # tagged logger is.
+  #
+  # @param signal [GenevaDrive::Signal, nil] the attached signal
+  # @yield the block to execute with the signal in scope
+  # @return [Object] the result of the block
+  # @api private
+  public def with_received_signal(signal)
+    previous_received_signal = @received_signal
+    @received_signal = signal
+    yield
+  ensure
+    @received_signal = previous_received_signal
+  end
+
   # Returns the Logger properly tagged to this Workflow
   #
   # @return [Logger]

data/lib/geneva_drive.rb

@@ -32,6 +32,8 @@ module GenevaDrive
   autoload :StepExecutionError, "geneva_drive/flow_control"
   autoload :StepNotDefinedError, "geneva_drive/flow_control"
   autoload :CursorTooLargeError, "geneva_drive/flow_control"
+  autoload :SignalPayloadTooLargeError, "geneva_drive/flow_control"
+  autoload :WorkflowNotOngoing, "geneva_drive/flow_control"
   autoload :StepFailedError, "geneva_drive/flow_control"
   autoload :PreconditionError, "geneva_drive/flow_control"
   autoload :FlowControl, "geneva_drive/flow_control"
@@ -40,6 +42,8 @@ module GenevaDrive
   autoload :StepCollection, "geneva_drive/step_collection"
   autoload :Workflow, "geneva_drive/workflow"
   autoload :StepExecution, "geneva_drive/step_execution"
+  autoload :Signal, "geneva_drive/signal"
+  autoload :SignalMatcher, "geneva_drive/signal_matcher"
   autoload :IterableStep, "geneva_drive/iterable_step"
   autoload :Executor, "geneva_drive/executor"
   autoload :ExceptionPolicy, "geneva_drive/exception_policy"
@@ -88,6 +92,22 @@ module GenevaDrive
     # @return [Integer, nil]
     attr_accessor :max_cursor_size
 
+    # Maximum size in bytes of a signal payload once serialized to JSON.
+    # A payload describes an event - an order id, an amount, a decision -
+    # not the data the step is going to work on. Exceeding the limit raises
+    # SignalPayloadTooLargeError from Workflow#signal!, before anything is
+    # persisted. Set to nil to disable the check.
+    # @return [Integer, nil]
+    attr_accessor :max_signal_payload_size
+
+    # How long a step execution may sit in the "waiting" state before
+    # housekeeping counts it in the `geneva_drive.waiting_overdue` gauge.
+    # Waiting indefinitely is legitimate, so this is not a timeout - it is
+    # the floor that keeps a stalled rendezvous from being silent.
+    # Set to nil to disable the gauge.
+    # @return [ActiveSupport::Duration, nil]
+    attr_accessor :waiting_visibility_threshold
+
     # Whether to defer job enqueueing to after the database transaction commits.
     # When true (the default in non-test environments), jobs are enqueued inside
     # an `after_all_transactions_commit` callback to ensure the step execution
@@ -175,5 +195,7 @@ module GenevaDrive
   self.housekeeping_batch_size = 1000
   self.stuck_recovery_action = :reattempt
   self.max_cursor_size = 128 * 1024
+  self.max_signal_payload_size = 128 * 1024
+  self.waiting_visibility_threshold = 7.days
   self.enqueue_after_commit = !Rails.env.test?
 end