pgbus 0.20.1 → 0.21.1

data/lib/pgbus/workflow/step_definition.rb

@@ -15,8 +15,11 @@ module Pgbus
       # (job_options:) may set. Anything else raises at class load.
       JOB_OPTIONS = %i[queue priority wait wait_until].freeze
 
+      # wait_timeout is the static wait_timeout: in seconds (nil for a per-run
+      # one); wait_timeout_callable the per-run one as given, a lambda or a
+      # method name (issue #574).
       attr_reader :name, :body, :wait, :skip_if, :job_options, :timeout, :exception_policies,
-                  :wait_for, :wait_for_name, :wait_timeout, :on_timeout
+                  :wait_for, :wait_for_name, :wait_timeout, :wait_timeout_callable, :on_timeout
 
       def self.validate_job_options!(options)
         raise ArgumentError, "job_options: must be a Hash, got #{options.inspect}" unless options.is_a?(Hash)
@@ -104,6 +107,44 @@ module Pgbus
         result ? true : false
       end
 
+      # True when wait_timeout: is a lambda or a method name, evaluated per run.
+      def dynamic_wait_timeout? = !wait_timeout_callable.nil?
+
+      # The timeout_at of the step parking now on +workflow+: now plus a
+      # static wait_timeout:, what a per-run one returns (a deadline already
+      # past is now, so the next sweep applies on_timeout), or nil.
+      def timeout_at_for(workflow, now)
+        return now + wait_timeout if wait_timeout
+        return nil unless dynamic_wait_timeout?
+
+        deadline = resolve_wait_timeout(workflow)
+        deadline.is_a?(Time) ? [deadline, now].max : deadline && (now + deadline)
+      end
+
+      # What the per-run wait_timeout: returns on +workflow+: a Time (an
+      # absolute deadline), seconds from now, or nil (no timeout). It runs as
+      # a guard does (a flow-control verb raises NotInStep), in a savepoint:
+      # a query of its own that fails in the database must not abort the
+      # run's transaction, which the step's outcome is written in. Anything
+      # else raises ArgumentError, as an error it raises goes up, for the
+      # step's exception policies.
+      def resolve_wait_timeout(workflow)
+        callable = wait_timeout_callable
+        value = WorkflowEntry.transaction(requires_new: true) do
+          workflow.send(:guarding) { callable.is_a?(Symbol) ? workflow.send(callable) : workflow.instance_exec(&callable) }
+        end
+        return nil if value.nil?
+        if value.is_a?(Numeric) || self.class.duration?(value)
+          return self.class.seconds(value, "wait_timeout (step #{name})", positive: true)
+        end
+
+        time = value.to_time if value.respond_to?(:to_time) && !value.is_a?(String)
+        return time if time.is_a?(Time)
+
+        raise ArgumentError, "wait_timeout: of step #{name} must return a Time, a Duration, a number of seconds or nil, " \
+                             "got #{value.inspect}"
+      end
+
       private
 
       def assign_wait(wait_for, wait_timeout, on_timeout)
@@ -112,8 +153,9 @@ module Pgbus
         raise ArgumentError, "on_timeout: needs wait_for: and wait_timeout:" if !on_timeout.nil? && (@wait_for.nil? || wait_timeout.nil?)
         raise ArgumentError, "wait_timeout: needs wait_for: (it bounds the wait for a signal)" if wait_timeout && @wait_for.nil?
 
-        @wait_timeout = self.class.seconds(wait_timeout, "wait_timeout", positive: true)
-        @on_timeout = @wait_timeout && normalize_on_timeout(on_timeout.nil? ? :pause! : on_timeout)
+        @wait_timeout_callable = wait_timeout if wait_timeout.is_a?(Symbol) || wait_timeout.is_a?(Proc)
+        @wait_timeout = @wait_timeout_callable ? nil : self.class.seconds(wait_timeout, "wait_timeout", positive: true)
+        @on_timeout = (@wait_timeout || @wait_timeout_callable) && normalize_on_timeout(on_timeout.nil? ? :pause! : on_timeout)
       end
 
       def normalize_on_timeout(value)

data/lib/pgbus/workflow/step_execution.rb

@@ -13,7 +13,8 @@ module Pgbus
     #    heartbeat is on: its worker died, and the stall action applies now.
     # 2. Guards, still under the lock: the class or the step no longer
     #    defined (pause with StepNotDefined, or skip a removed_step),
-    #    cancel_if, skip_if.
+    #    cancel_if, skip_if (which read woken? and signal_ready? through the
+    #    attempt's SignalPeek), then a wait_for: step's signal gate.
     # 3. The body, outside any transaction, inside `catch` for the
     #    flow-control verbs and `rescue StandardError` for the policies.
     # 4. The outcome, under the lock again (StepOutcome).
@@ -21,7 +22,9 @@ module Pgbus
     # Whatever the step's outcome, the job itself completes normally: the
     # step row and the pgbus.workflow.* events carry the truth.
     class StepExecution
-      Claim = Data.define(:record, :step, :klass, :definition, :workflow, :guard_error)
+      # +adopt_held+: a guard raised before the gate, so the signal the step
+      # holds is put on this attempt's row before its outcome (Signals.adopt_held).
+      Claim = Data.define(:record, :step, :klass, :definition, :workflow, :guard_error, :adopt_held)
 
       def initialize(step_id, delivery: Pgbus::ActiveJob::Delivery.current, config: Pgbus.configuration)
         @step_id = step_id
@@ -103,15 +106,23 @@ module Pgbus
         return undefined!(record, step, notes, "#{record.workflow_class} is not a Pgbus::Workflow class any more") unless klass
 
         unless definition
-          return skip!(record, step, notes, klass) if klass.workflow_steps.removed?(step.step_name)
+          if klass.workflow_steps.removed?(step.step_name)
+            # The definition is gone, so whether it waited is unknown: look for a
+            # held signal only where the signal schema exists.
+            Signals.adopt_held(step, WorkflowSignal.held_by_step(record.workflow_id, step.step_name)) if Sweep.waiting_ready?
+            return skip!(record, step, notes, klass)
+          end
 
           return undefined!(record, step, notes, "#{klass.name} has no step #{step.step_name} " \
                                                  "(declare `removed_step :#{step.step_name}` to skip it)")
         end
 
         workflow = klass.new(record)
-        workflow.send(:enter_step, StepContext::Attempt.new(step_id: step.id, step_name: step.step_name, attempt: step.attempt))
-        claim = Claim.new(record: record, step: step, klass: klass, definition: definition, workflow: workflow, guard_error: nil)
+        peek = SignalPeek.new(record, step, definition)
+        workflow.send(:enter_step, StepContext::Attempt.new(step_id: step.id, step_name: step.step_name, attempt: step.attempt),
+                      signals: peek)
+        claim = Claim.new(record: record, step: step, klass: klass, definition: definition, workflow: workflow, guard_error: nil,
+                          adopt_held: false)
         begin
           verdict = workflow.send(:guarding) do
             next FlowControl::Outcome.cancel if cancel?(klass, workflow)
@@ -119,20 +130,23 @@ module Pgbus
             FlowControl::Outcome.skip if definition.skip?(workflow)
           end
         rescue StandardError => e
-          return claim.with(guard_error: e)
+          return claim.with(guard_error: e, adopt_held: definition.waits_for_signal?)
+        end
+        if verdict
+          Signals.adopt_held(step, peek.held)
+          return outcome!(record, step, notes, klass, verdict)
         end
-        return outcome!(record, step, notes, klass, verdict) if verdict
 
-        gate(record, step, definition, workflow, claim)
+        gate(record, step, definition, workflow, claim, peek)
       end
 
       # A wait_for: step runs only with a signal (or once its wait timed
       # out); without one it parks and this delivery ends. A matcher that
       # raises is handed to the exception policies, like a guard.
-      def gate(record, step, definition, workflow, claim)
+      def gate(record, step, definition, workflow, claim, peek)
         return claim unless definition.waits_for_signal?
 
-        Signals.gate(record, step, definition, workflow) ? claim : nil
+        Signals.gate(record, step, definition, workflow, peek: peek) ? claim : nil
       rescue StandardError => e
         claim.with(guard_error: e)
       end
@@ -185,6 +199,9 @@ module Pgbus
           next park_fan_out(record, claim, outcome, notes) if outcome&.action == :fan_out
 
           notes.report(decision.handler_error, handler_context(claim)) if decision&.handler_error
+          # Here rather than in the claim's transaction, which the statement
+          # a guard failed on may have aborted.
+          Signals.adopt_held(claim.step, SignalPeek.new(record, claim.step, claim.definition).held) if claim.adopt_held
           StepOutcome.apply(record, claim.step, outcome, notes: notes, error: error, output: claim.workflow.pending_output,
                                                          report: decision ? decision.report : true, duration: duration,
                                                          klass: claim.klass)

data/lib/pgbus/workflow/wait_timeout.rb

@@ -14,7 +14,8 @@ module Pgbus
     #            received_signal nil and timed_out? true
     #   :notify  only the event and the instrumentation (issue #565): the
     #            same attempt keeps waiting and timeout_at moves to its next
-    #            deadline, one wait_timeout on (see next_deadline)
+    #            deadline, one wait_timeout on (see next_deadline); a per-run
+    #            wait_timeout: is evaluated again for it (see notify_deadline)
     #
     # A signal that arrives after the timeout fired stays pending, like any
     # signal no step waits for, except under :notify, whose step still waits
@@ -56,12 +57,12 @@ module Pgbus
                                                            timeout_at: step.timeout_at&.utc&.iso8601(3) }.compact)
         notes.instrument("pgbus.workflow.wait_timed_out",
                          Runner.payload_for(record, step_name: step.step_name, attempt: step.attempt, action: action.to_s))
-        apply(record, step, action, notes: notes, now: now, wait_timeout: definition&.wait_timeout)
+        apply(record, step, action, notes: notes, now: now, definition: definition)
         action
       end
 
-      def apply(record, step, action, notes:, now:, wait_timeout: nil)
-        return notify!(step, wait_timeout, now: now) if action == :notify
+      def apply(record, step, action, notes:, now:, definition: nil)
+        return notify!(record, step, definition, notes: notes, now: now) if action == :notify
         return Waiting.wake!(record, step, now: now, timeout_at: [step.timeout_at, now].compact.min) if action == :run
 
         error = action == :pause! ? timed_out_error(step, now) : nil
@@ -69,12 +70,42 @@ module Pgbus
       end
 
       # The run and the step row stay as they are (no outcome, no StepJob);
-      # only timeout_at moves.
-      def notify!(step, wait_timeout, now:)
-        timeout_at = next_deadline(step.timeout_at, wait_timeout, now)
+      # only timeout_at moves. A per-run wait_timeout: that raises here, or
+      # returns what is no deadline, pauses the run with that error: the
+      # sweep holds the run's row lock, under which no on_exception block
+      # may run, so the exception policies do not apply.
+      def notify!(record, step, definition, notes:, now:)
+        timeout_at = notify_deadline(record, step, definition, now)
+      rescue StandardError => e
+        Waiting.resolve!(record, step, FlowControl::Outcome.pause, notes: notes, error: e)
+      else
         step.timeout_at = timeout_at if WorkflowStep.move_deadline!(step.id, timeout_at: timeout_at).positive?
       end
 
+      # The next deadline of a :notify wait. A per-run wait_timeout: (issue
+      # #574) is evaluated again: a Time after now is the next deadline, a
+      # duration moves it one period on as a static one does, and nil or a
+      # Time not after now leaves the wait without a deadline (it was
+      # reported once and keeps waiting).
+      def notify_deadline(record, step, definition, now)
+        return next_deadline(step.timeout_at, definition.wait_timeout, now) unless definition.dynamic_wait_timeout?
+
+        deadline = definition.resolve_wait_timeout(instance_for(record, step))
+        case deadline
+        when nil then nil
+        when Time then deadline if deadline > now
+        else next_deadline(step.timeout_at, deadline, now)
+        end
+      end
+
+      # The workflow instance a per-run wait_timeout: is evaluated on, as the
+      # StepJob builds it for the attempt.
+      def instance_for(record, step)
+        record.workflow_klass.new(record).tap do |workflow|
+          workflow.send(:enter_step, StepContext::Attempt.new(step_id: step.id, step_name: step.step_name, attempt: step.attempt))
+        end
+      end
+
       # The first deadline after +now+ on the wait's wait_timeout grid: one
       # wait_timeout on when the sweep is within a period of the deadline
       # (and for expire_wait ahead of it). A sweep that missed several

data/lib/pgbus/workflow.rb

@@ -111,12 +111,18 @@ module Pgbus
       #
       #   SignupWorkflow.start!(user, plan: "pro")    # arguments["plan"]
       #   CheckoutWorkflow.start!(order, idempotency_key: "checkout-#{order.id}")
+      #   ReportWorkflow.start!(actor: "alice", reason: "monthly close")
       #
-      # +idempotency_key+ is reserved: with one, a second start! with the same
-      # key returns the first run (any status) with duplicate_start? true.
-      # Raises AlreadyRunning when the subject already has an ongoing run.
-      def start!(subject = nil, idempotency_key: nil, **arguments)
-        Runner.start(self, subject: subject, arguments: arguments, idempotency_key: idempotency_key)
+      # +idempotency_key+, +actor+ and +reason+ are reserved and never reach
+      # the arguments. With an idempotency_key, a second start! with the same
+      # key returns the first run (any status) with duplicate_start? true and
+      # records nothing. +actor+ (default "app") and +reason+ go on the
+      # started event, and the actor on the first step_scheduled, as on every
+      # outside verb. Raises AlreadyRunning when the subject already has an
+      # ongoing run.
+      def start!(subject = nil, idempotency_key: nil, actor: nil, reason: nil, **arguments)
+        Runner.start(self, subject: subject, arguments: arguments, idempotency_key: idempotency_key,
+                           actor: actor || Runner::DEFAULT_ACTOR, reason: reason)
       end
 
       # The run with this workflow_id, as an instance of the class that

data/sig/pgbus/client.rbs

@@ -13,7 +13,7 @@ module Pgbus
     def convert_archive_partitioned: (untyped queue_name, ?partition_interval: untyped, ?retention_interval: untyped, ?leading_partition: untyped) -> untyped
     def create_fifo_index: (untyped queue_name) -> untyped
     def create_fifo_indexes_all: () -> untyped
-    def dead_letter_physical_name: (untyped queue_name) -> untyped
+    def dead_letter_physical_name: (untyped queue_name, ?prefixed: untyped) -> untyped
     def delete_batch: (untyped queue_name, untyped msg_ids, ?prefixed: untyped) -> untyped
     def delete_message: (untyped queue_name, untyped msg_id, ?prefixed: untyped) -> untyped
     def drop_queue: (untyped queue_name, ?prefixed: untyped) -> untyped
@@ -25,10 +25,12 @@ module Pgbus
     def in_recovery?: () -> bool
     def list_notify_insert_throttles: () -> untyped
     def list_queues: () -> untyped
-    def message_archived?: (untyped queue_name, msg_id: untyped) -> bool
-    def message_exists?: (untyped queue_name, ?msg_id: untyped, ?uniqueness_key: untyped) -> bool
-    def message_in_queue?: (untyped queue_name, msg_id: untyped) -> bool
-    def message_with_job_id?: (untyped queue_name, job_id: untyped) -> bool
+    def locate_message: (untyped queue_name, msg_id: untyped, ?job_id: untyped, ?prefixed: untyped) -> String?
+    def locate_messages: (untyped queue_name, Array[[untyped, untyped]] messages, ?prefixed: untyped) -> Hash[Integer, String]
+    def message_archived?: (untyped queue_name, msg_id: untyped, ?prefixed: untyped) -> bool?
+    def message_exists?: (untyped queue_name, ?msg_id: untyped, ?uniqueness_key: untyped, ?prefixed: untyped) -> bool?
+    def message_in_queue?: (untyped queue_name, msg_id: untyped, ?prefixed: untyped) -> bool?
+    def message_with_job_id?: (untyped queue_name, job_id: untyped, ?prefixed: untyped) -> bool?
     def metrics: (?untyped queue_name) -> untyped
     def move_to_dead_letter: (untyped queue_name, untyped message, ?headers: untyped) -> untyped
     def notify_enabled?: (untyped queue_name) -> bool

data/sig/pgbus.rbs

@@ -56,6 +56,49 @@ module Pgbus
     end
   end
 
+  # Partial: only the readers a workflow step's guards and body use to see
+  # its signal (issue #572), and a step definition's wait_timeout: (issue
+  # #574). The rest of Workflow is not covered by spec/pgbus/rbs_spec.rb yet.
+  class Workflow
+    include Instance
+
+    class Error < Pgbus::Error
+    end
+
+    # A step reader (woken?, signal_ready?, ...) called outside a step.
+    class NotInStep < Error
+    end
+
+    module Instance
+      # The Pgbus::WorkflowSignal a wait_for: step took, or nil when its wait timed out.
+      def received_signal: () -> untyped
+      # The body runs because the wait timed out (on_timeout: :run).
+      def timed_out?: () -> bool
+      # This attempt holds a signal before its gate: signal! woke it, or an
+      # earlier attempt took one and reattempted or paused.
+      def woken?: () -> bool
+      # The gate runs this attempt with a signal: the held one, or a pending
+      # one its wait_for: matcher takes.
+      def signal_ready?: () -> bool
+    end
+
+    # One declared step (klass.workflow_steps.fetch(:name)): its wait_timeout:.
+    class StepDefinition
+      # A static wait_timeout: in seconds; nil without one, or for a per-run one.
+      def wait_timeout: () -> Numeric?
+      # A per-run wait_timeout: as declared: a method name or a lambda.
+      def wait_timeout_callable: () -> (Symbol | Proc)?
+      # wait_timeout: is a method name or a lambda, evaluated per run.
+      def dynamic_wait_timeout?: () -> bool
+      # The timeout_at of the step parking at +now+ on +workflow+; nil for no timeout.
+      # +now+ and the result are a Time or an ActiveSupport::TimeWithZone (Time.current
+      # in an app with a time zone), which sig/ does not declare: untyped.
+      def timeout_at_for: (Workflow workflow, untyped now) -> untyped
+      # What a per-run wait_timeout: returns on +workflow+: a deadline, seconds from now, or nil.
+      def resolve_wait_timeout: (Workflow workflow) -> (Time | Numeric)?
+    end
+  end
+
   def self.configuration: () -> Configuration
   def self.configuration_snapshot: () -> Configuration::Snapshot
   def self.configure: () { (Configuration) -> void } -> Configuration

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pgbus
 version: !ruby/object:Gem::Version
-  version: 0.20.1
+  version: 0.21.1
 platform: ruby
 authors:
 - Mikael Henriksson
@@ -588,6 +588,7 @@ files:
 - lib/pgbus/workflow/removed_step.rb
 - lib/pgbus/workflow/runner.rb
 - lib/pgbus/workflow/signal_matcher.rb
+- lib/pgbus/workflow/signal_peek.rb
 - lib/pgbus/workflow/signals.rb
 - lib/pgbus/workflow/states.rb
 - lib/pgbus/workflow/step_context.rb