pgbus 0.19.3 → 0.20.0

data/lib/pgbus/workflow/fan_out_definition.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # A `fan_out` step (issue #541): a StepDefinition whose body adds jobs
+    # to a builder instead of doing the work itself. FanOut runs the body,
+    # opens the batch and parks the step on it.
+    class FanOutDefinition < StepDefinition
+      # What a batch with dead-lettered members does to the run.
+      FAILURE_ACTIONS = %i[pause! cancel! skip!].freeze
+      # The step options a fan_out takes (on_batch_failure: besides): every
+      # one but the signal waits.
+      OPTIONS = (StepDefinition::OPTIONS - %i[wait_for wait_timeout on_timeout]).freeze
+
+      attr_reader :on_batch_failure
+
+      def initialize(on_batch_failure: :pause!, **)
+        @on_batch_failure = normalize_failure_action(on_batch_failure)
+        super(**)
+      end
+
+      def kind = "fan_out"
+
+      # Runs the body with a recording builder, opens the batch, and ends the
+      # attempt with the :fan_out outcome (FanOut.run throws it).
+      def invoke(workflow)
+        FanOut.run(self, workflow)
+      end
+
+      private
+
+      def normalize_failure_action(value)
+        verb = value.nil? ? nil : :"#{value.to_s.delete_suffix("!")}!"
+        return verb if FAILURE_ACTIONS.include?(verb)
+
+        raise ArgumentError,
+              "on_batch_failure: must be one of #{FAILURE_ACTIONS.map(&:inspect).join(", ")}, got #{value.inspect}"
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/flow_control.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The flow-control verbs. Inside a step body each one ends the body at
+    # once (`throw :pgbus_workflow_flow, Outcome`) and the StepJob applies the
+    # outcome under the workflow lock. Outside a step, `pause!` and `cancel!`
+    # are operator verbs (they change the run through the Runner, with
+    # actor:/reason: on the audit trail); the step-only verbs raise NotInStep.
+    module FlowControl
+      THROW_TAG = :pgbus_workflow_flow
+      ACTIONS = %i[complete reattempt pause cancel skip finish fan_out].freeze
+
+      # What a step attempt ended with. +wait+ is the reattempt delay in
+      # seconds (nil = now); +actor+ and +reason+ are what a step's own
+      # pause!/cancel! (or an operator's skip!) passed, for the audit trail;
+      # +details+ carries a fan_out's batch_id and size.
+      Outcome = Data.define(:action, :wait, :actor, :reason, :details) do
+        def initialize(action:, wait: nil, actor: nil, reason: nil, details: nil) = super
+
+        def self.complete = new(action: :complete, wait: nil)
+        def self.pause(actor: nil, reason: nil) = new(action: :pause, actor: actor, reason: reason)
+        def self.cancel(actor: nil, reason: nil) = new(action: :cancel, actor: actor, reason: reason)
+        def self.skip(actor: nil, reason: nil) = new(action: :skip, actor: actor, reason: reason)
+        def self.finish = new(action: :finish, wait: nil)
+        def self.reattempt(wait: nil) = new(action: :reattempt, wait: StepDefinition.seconds(wait, "wait", positive: false))
+        # A fan_out body returned and its batch is open: the step parks on it.
+        def self.fan_out(batch_id:, size:) = new(action: :fan_out, details: { batch_id: batch_id, size: size }.freeze)
+
+        # An ExceptionPolicy action (:pause!, :reattempt!, :cancel!, :skip!).
+        def self.for(verb, wait: nil)
+          case verb.to_s.delete_suffix("!").to_sym
+          when :pause then pause
+          when :cancel then cancel
+          when :skip then skip
+          when :reattempt then reattempt(wait: wait)
+          when :finished, :finish then finish
+          else raise ArgumentError, "unknown flow-control verb #{verb.inspect}"
+          end
+        end
+
+        def reattempt? = action == :reattempt
+      end
+
+      # Inside a step: end it here, completed, with the workflow paused.
+      # Outside: pause the run (ready, performing or waiting → paused).
+      def pause!(actor: nil, reason: nil)
+        return throw_outcome(Outcome.pause(actor: actor, reason: reason)) if in_step?
+
+        apply_outside_verb(:pause, actor: actor, reason: reason)
+      end
+
+      # Inside a step: end it here and cancel the workflow.
+      # Outside: cancel the run from any ongoing state.
+      def cancel!(actor: nil, reason: nil)
+        return throw_outcome(Outcome.cancel(actor: actor, reason: reason)) if in_step?
+
+        apply_outside_verb(:cancel, actor: actor, reason: reason)
+      end
+
+      # Ends this attempt as failed and schedules the next one, +wait+
+      # seconds from now. Step bodies only.
+      def reattempt!(wait: nil)
+        throw_outcome(Outcome.reattempt(wait: wait), verb: :reattempt!)
+      end
+
+      # Inside a step: end it here, skipped; the workflow moves on.
+      # Outside: skip the step a waiting run waits on (a signal it holds is
+      # consumed) and schedule the next one.
+      def skip!(actor: nil, reason: nil)
+        return throw_outcome(Outcome.skip) if in_step?
+
+        apply_outside_verb(:skip, actor: actor, reason: reason)
+      end
+
+      # Ends the step as completed and finishes the workflow, skipping the
+      # steps after it. Step bodies only.
+      def finished!
+        throw_outcome(Outcome.finish, verb: :finished!)
+      end
+
+      private
+
+      def in_step? = !step_context.nil?
+
+      def throw_outcome(outcome, verb: nil)
+        raise NotInStep, "#{verb} can only be called inside a step body" if verb && !in_step?
+        raise NotInStep, "flow-control verbs end a step from its body or an on_exception block, not from cancel_if or skip_if" if @guarding
+
+        throw THROW_TAG, outcome
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/instance.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The instance side of a workflow: the run's readers, its outputs and
+    # idempotency keys, and the outside verbs. An instance wraps one
+    # Pgbus::WorkflowEntry row; inside a step it also knows which step
+    # attempt it is running (the step context).
+    module Instance
+      def workflow_id = record.workflow_id
+
+      def workflow_class = record.workflow_class
+
+      def status = record.status
+
+      def current_step = record.current_step
+
+      # True when start!(…, idempotency_key:) found an existing run with that
+      # key and returned it instead of starting another.
+      def duplicate_start? = @duplicate_start
+
+      # The record the run was started for, loaded through its GlobalID;
+      # nil for a subject-less run or when the record is gone.
+      def subject
+        return @subject if defined?(@subject)
+
+        @subject = Payload.locate_subject(record.subject_gid)
+      end
+
+      # start!'s keyword arguments, with indifferent access.
+      def arguments
+        return @arguments if defined?(@arguments)
+
+        @arguments = Payload.load(record.arguments) || {}.with_indifferent_access
+      end
+
+      def reload
+        @record = WorkflowEntry.find_by!(workflow_id: workflow_id)
+        remove_instance_variable(:@subject) if defined?(@subject)
+        remove_instance_variable(:@arguments) if defined?(@arguments)
+        self
+      end
+
+      # The audit trail, oldest first.
+      def events = WorkflowEvent.where(workflow_id: workflow_id).order(:id).to_a
+
+      # The outside verbs legal in the current status (States).
+      def available_actions = States.available_actions(status)
+
+      # Inside a step: record durable output values. They are written in the
+      # same UPDATE that completes the step, so they exist exactly when the
+      # step does; later steps read them with output_of. Merges across calls.
+      def output(**values)
+        context = step_context!(:output)
+        raise NotInStep, "output can only be called from the step body, not from cancel_if or skip_if" if @guarding
+
+        merged = (@output_values || {}).merge(values)
+        @pending_output = Payload.dump(merged, what: "output of step #{context.step_name}")
+        @output_values = merged
+        nil
+      end
+
+      # What this attempt's output(...) calls recorded, serialized (or nil).
+      def pending_output = @pending_output
+
+      # The output of +step_name+'s latest completed attempt, or nil. A
+      # failed or reattempted attempt's output is never returned.
+      def output_of(step_name)
+        Payload.load(WorkflowStep.completed_output(workflow_id, step_name))
+      end
+
+      # { "step_name" => output } for every step with a completed attempt.
+      def outputs
+        WorkflowStep.completed_outputs(workflow_id).transform_values { |output| Payload.load(output) }
+      end
+
+      # The same on every attempt of this step (and on a resume_at! re-run):
+      # hand it to an external API so a repeated side effect is deduplicated.
+      def step_idempotency_key
+        "pgbus-wf:#{workflow_id}:#{step_context!(:step_idempotency_key).step_name}"
+      end
+
+      # Inside a wait_for: step: the Pgbus::WorkflowSignal the step took (its
+      # +payload+ has indifferent access), or nil when its wait timed out.
+      def received_signal
+        step_context!(:received_signal)
+        @received_signal
+      end
+
+      # Inside a wait_for: step: true when the body runs because the wait
+      # timed out (on_timeout: :run) rather than for a signal.
+      def timed_out?
+        step_context!(:timed_out?)
+        @timed_out == true
+      end
+
+      # Deliver a signal to this run. Stored under the run's lock; a step
+      # waiting for it (wait_for:) is woken at once, otherwise it waits,
+      # pending, for the step that claims it. With an idempotency_key a
+      # redelivery stores nothing and returns the first signal
+      # (duplicate_delivery? true). Raises NotOngoing on a finished or
+      # cancelled run. +reason+ goes on the signal_received event.
+      #
+      #   wf.signal!(:payment_confirmed, payload: { amount: 42 }, idempotency_key: event.id)
+      def signal!(name, payload: {}, idempotency_key: nil, actor: nil, reason: nil)
+        @record, signal = Signals.deliver(workflow_id, name, payload: payload, idempotency_key: idempotency_key,
+                                                             actor: actor || Runner::DEFAULT_ACTOR, reason: reason)
+        signal
+      end
+
+      # Differs per attempt, for the rare call that must not be deduplicated
+      # across attempts.
+      def attempt_idempotency_key
+        "#{step_idempotency_key}:#{step_context!(:attempt_idempotency_key).attempt}"
+      end
+
+      # paused → ready. A failed step runs again as a new attempt, a
+      # completed one is followed by the next step.
+      def resume!(actor: nil, reason: nil) = apply_outside_verb(:resume, actor: actor, reason: reason)
+
+      # paused → ready, running +step_name+ next.
+      def resume_at!(step_name, actor: nil, reason: nil)
+        apply_outside_verb(:resume_at, actor: actor, reason: reason, step_name: step_name)
+      end
+
+      # The scheduled step runs now instead of after its wait.
+      def run_now!(actor: nil, reason: nil) = apply_outside_verb(:run_now, actor: actor, reason: reason)
+
+      # The scheduled step runs at +at+ instead.
+      def reschedule!(at:, actor: nil, reason: nil) # rubocop:disable Naming/MethodParameterName
+        apply_outside_verb(:reschedule, actor: actor, reason: reason, at: at)
+      end
+
+      # A finished or cancelled run, started again as a new run with the same
+      # subject and arguments. Returns the new run.
+      def rerun!(actor: nil, reason: nil)
+        Runner.rerun(self, actor: actor || Runner::DEFAULT_ACTOR, reason: reason)
+      end
+
+      def inspect
+        "#<#{self.class.name} workflow_id=#{workflow_id.inspect} status=#{status.inspect} current_step=#{current_step.inspect}>"
+      end
+
+      private
+
+      attr_reader :step_context
+
+      def apply_outside_verb(verb, actor:, reason:, **)
+        @record = Runner.apply(self, verb, actor: actor || Runner::DEFAULT_ACTOR, reason: reason, **)
+        self
+      end
+
+      def step_context!(reader)
+        step_context || raise(NotInStep, "#{reader} can only be called inside a step body")
+      end
+
+      # Used by the StepJob only: cancel_if and skip_if run inside this, where
+      # the flow-control verbs raise NotInStep (only the body and an
+      # on_exception block can end a step).
+      def guarding
+        @guarding = true
+        yield
+      ensure
+        @guarding = false
+      end
+
+      # Used by the StepJob only.
+      def enter_step(context)
+        @step_context = context
+        @output_values = nil
+        @pending_output = nil
+        @received_signal = nil
+        @timed_out = false
+      end
+
+      # Used by the StepJob's signal gate only.
+      def enter_wait(signal:, timed_out:)
+        @received_signal = signal
+        @timed_out = timed_out
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/outside_verbs.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # The outside verbs, each called by Runner.apply with the run's row
+    # locked and the transition already checked against the States table.
+    module OutsideVerbs
+      module_function
+
+      def pause(record, notes, actor:, reason:)
+        from = record.status
+        Runner.transition!(record, :pause, paused_at: Time.current)
+        Runner.event!(record, "paused", actor: actor, reason: reason, details: { from: from })
+        notes.instrument("pgbus.workflow_paused", Runner.payload_for(record, step_name: record.current_step, error_class: nil))
+      end
+
+      def cancel(record, notes, actor:, reason:)
+        now = Time.current
+        from = record.status
+        Runner.transition!(record, :cancel, cancelled_at: now)
+        WorkflowStep.cancel_open!(record.workflow_id, now: now)
+        Runner.event!(record, "cancelled", actor: actor, reason: reason, details: { from: from })
+        notes.instrument("pgbus.workflow_cancelled", Runner.payload_for(record, step_name: record.current_step))
+      end
+
+      # What runs next depends on the row the run paused on: a failed one
+      # (an exception or a stall) runs again as a new attempt; a completed or
+      # skipped one (the step called pause!) is followed by the next step, or
+      # finishes the run when it was the last; a scheduled one (an outside
+      # pause! before it ran) is sent again; one still running goes on; one
+      # still waiting (for a signal or its batch) waits again.
+      def resume(record, notes, actor:, reason:)
+        latest = WorkflowStep.latest_for(record.workflow_id)
+        resumed = { paused_at: nil, **ErrorCapture.cleared }
+        case latest&.status
+        when "in_progress"
+          Runner.transition!(record, :unpause, **resumed)
+          resumed_event(record, actor, reason, latest)
+        when "waiting"
+          extend_wait(record, latest)
+          Runner.transition!(record, :rewait, **resumed)
+          resumed_event(record, actor, reason, latest)
+        when "scheduled"
+          Runner.transition!(record, :resume, **resumed)
+          resumed_event(record, actor, reason, latest)
+          latest.update!(enqueued_at: Time.current)
+        when "failed"
+          Runner.transition!(record, :resume, **resumed)
+          resumed_event(record, actor, reason, latest)
+          Runner.schedule!(record, latest.step_name, at: Time.current, actor: actor)
+        else
+          resume_after(record, notes, latest, resumed, actor: actor, reason: reason)
+        end
+      end
+
+      # The step a waiting run waits on is skipped (a signal it holds is
+      # consumed), as if it had called skip!; the next step is scheduled, or
+      # the run finishes after the last. Its fan-out batch, if any, runs on and
+      # its callback finds nothing to resume.
+      def skip(record, notes, actor:, reason:)
+        step = WorkflowStep.latest_for(record.workflow_id)
+        unless step&.status == "waiting"
+          raise InvalidTransition, "workflow #{record.workflow_id} has no waiting step to skip (latest: #{step&.status || "none"})"
+        end
+
+        Waiting.resolve!(record, step, FlowControl::Outcome.skip(actor: actor, reason: reason), notes: notes)
+      end
+
+      def resume_at(record, _notes, actor:, reason:, step_name:)
+        definition = definition!(record, step_name)
+        now = Time.current
+        WorkflowStep.cancel_open!(record.workflow_id, now: now)
+        Runner.transition!(record, :resume_at, current_step: definition.name, paused_at: nil, **ErrorCapture.cleared)
+        Runner.event!(record, "resumed", actor: actor, reason: reason, details: { resume_at: definition.name })
+        Runner.schedule!(record, definition.name, at: now, actor: actor)
+      end
+
+      def run_now(record, _notes, actor:, reason:)
+        reschedule(record, nil, actor: actor, reason: reason, at: Time.current, event: "run_now")
+      end
+
+      def reschedule(record, _notes, actor:, reason:, at:, event: "rescheduled") # rubocop:disable Naming/MethodParameterName
+        time = at.respond_to?(:to_time) && !at.is_a?(String) ? at.to_time : nil
+        raise ArgumentError, "reschedule!(at:) takes a Time, got #{at.inspect}" unless time
+
+        step = WorkflowStep.where(workflow_id: record.workflow_id, status: "scheduled").order(id: :desc).first
+        raise InvalidTransition, "workflow #{record.workflow_id} has no scheduled step to #{event.tr("_", " ")}" unless step
+
+        verb = event == "run_now" ? :run_now : :reschedule
+        from = step.scheduled_at
+        Runner.transition!(record, verb)
+        step.update!(scheduled_at: time, enqueued_at: Time.current)
+        Runner.event!(record, event, actor: actor, reason: reason, step: step,
+                                     details: { from: from.utc.iso8601(3), to: step.scheduled_at.utc.iso8601(3) })
+      end
+
+      def resume_after(record, notes, latest, resumed, actor:, reason:)
+        klass = record.workflow_klass || raise(StepNotDefined, "#{record.workflow_class} is not a Pgbus::Workflow class any more")
+        steps = klass.workflow_steps
+        if latest && steps.entries.none? { |entry| entry.name == latest.step_name }
+          raise StepNotDefined, "#{klass.name} has no step #{latest.step_name} (declare `removed_step :#{latest.step_name}`)"
+        end
+
+        following = latest ? steps.next_after(latest.step_name) : steps.first
+        if following.nil?
+          Runner.transition!(record, :finish, finished_at: Time.current, **resumed)
+          resumed_event(record, actor, reason, latest)
+          Runner.event!(record, "finished", actor: actor)
+          notes.instrument("pgbus.workflow_finished", Runner.payload_for(record))
+          return
+        end
+
+        Runner.transition!(record, :resume, current_step: following.name, **resumed)
+        resumed_event(record, actor, reason, latest)
+        Runner.schedule!(record, following.name, at: Time.current + following.wait, actor: actor)
+      end
+
+      # The time a run spent paused does not count against its waiting
+      # step's wait_timeout: the deadline moves by the length of the pause
+      # (from when the step started waiting, if that came later).
+      def extend_wait(record, step, now: Time.current)
+        return unless step.timeout_at && record.paused_at
+
+        paused_for = now - [record.paused_at, step.waiting_since].compact.max
+        step.update_columns(timeout_at: step.timeout_at + paused_for) if paused_for.positive?
+      end
+
+      def resumed_event(record, actor, reason, step)
+        Runner.event!(record, "resumed", actor: actor, reason: reason, step: step)
+      end
+
+      def definition!(record, step_name)
+        klass = record.workflow_klass || raise(StepNotDefined, "#{record.workflow_class} is not a Pgbus::Workflow class any more")
+        klass.workflow_steps.fetch(step_name) || raise(StepNotDefined, "#{klass.name} has no step #{step_name}")
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/payload.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Pgbus
+  class Workflow
+    # Arguments, step outputs and the subject, to and from the database.
+    # Values go through ActiveJob::Arguments (JSON, GlobalID-aware, never
+    # Marshal), are capped at config.workflow_max_payload_size, and come back
+    # with indifferent access. GlobalIDs are gated by
+    # config.allowed_global_id_models, as job arguments are.
+    module Payload
+      module_function
+
+      def dump(values, what:, limit: Pgbus.configuration.workflow_max_payload_size)
+        serialized = ::ActiveJob::Arguments.serialize([values.to_h]).first
+        size = JSON.generate(serialized).bytesize
+        raise PayloadTooLarge, "#{what} is #{size} bytes, over workflow_max_payload_size (#{limit})" if size > limit
+
+        serialized
+      end
+
+      def load(serialized, configuration: Pgbus.configuration)
+        return nil if serialized.nil?
+
+        Serializer.assert_job_global_ids_allowed!(serialized, configuration: configuration)
+        ::ActiveJob::Arguments.deserialize([serialized]).first.with_indifferent_access
+      end
+
+      # The subject columns of a run: the polymorphic type and id the
+      # one-ongoing index keys on, and the GlobalID +subject+ loads through.
+      def subject_attributes(subject)
+        return { subject_type: nil, subject_id: nil, subject_gid: nil } if subject.nil?
+        unless subject.respond_to?(:to_global_id) && subject.respond_to?(:id)
+          raise ArgumentError, "a workflow subject must be a GlobalID record (an ActiveRecord model), got #{subject.class}"
+        end
+
+        klass = subject.class
+        type = klass.respond_to?(:base_class) ? klass.base_class.name : klass.name
+        { subject_type: type, subject_id: subject.id.to_s, subject_gid: subject.to_global_id.to_s }
+      end
+
+      # nil when the run has no subject or the record is gone.
+      def locate_subject(gid, configuration: Pgbus.configuration)
+        return nil if gid.nil?
+
+        Serializer.locate_global_id(gid, configuration: configuration)
+      rescue ActiveRecord::RecordNotFound
+        nil
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/registry.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Pgbus
+  class Workflow
+    # Every Pgbus::Workflow subclass, registered from `inherited` (what the
+    # insights page lists). The list is a frozen Array replaced whole under a
+    # lock built at load (docs/concurrency.md): a reader never sees it change
+    # under it, and two classes defined at once are both kept.
+    #
+    # Classes are kept as objects, not names: `Class.new(Pgbus::Workflow)`
+    # has no name yet when `inherited` runs. Names are read when listing, and
+    # a name defined again (code reloading) lists its latest class only.
+    module Registry
+      @lock = Mutex.new
+      @classes = [].freeze
+
+      class << self
+        def register(klass)
+          @lock.synchronize { @classes = merged(@classes, klass) }
+          klass
+        end
+
+        def classes
+          named = @lock.synchronize { @classes }.select(&:name)
+          named.reverse.uniq(&:name).reverse
+        end
+
+        # Specs only.
+        def reset!
+          @lock.synchronize { @classes = [].freeze }
+        end
+
+        # SHA256 of the ordered [name, kind] pairs of a Steps list.
+        def digest(steps)
+          Digest::SHA256.hexdigest(JSON.generate(steps.digest_pairs))
+        end
+
+        private
+
+        # A class registered again by name (code reloading) replaces the old
+        # one, so reloads do not accumulate. Anonymous classes are kept until
+        # they get a name; classes lists them then.
+        def merged(list, klass)
+          name = klass.name
+          (list.reject { |existing| existing.equal?(klass) || (name && existing.name == name) } << klass).freeze
+        end
+      end
+    end
+  end
+end

data/lib/pgbus/workflow/removed_step.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Pgbus
+  class Workflow
+    # A removed_step gravestone: a step that existed once. A scheduled row
+    # still naming it is skipped and the workflow moves on to the next step
+    # declared after it.
+    RemovedStep = Data.define(:name) do
+      def kind = "removed"
+
+      def removed? = true
+    end
+  end
+end